From b42380ced6d3e485007b743634206d28f59c3142 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sat, 15 Aug 2026 03:11:19 +0200 Subject: [PATCH] shader_recompiler: add SSA --- CMakeLists.txt | 41 +- .../renderer/pipeline/descriptors.cpp | 4 +- .../renderer/pipeline/shaderResourceBarrier.h | 2 +- .../renderer/pipeline/shaderSubgroup.cpp | 2 +- .../host_gpu/renderer/renderCompute.cpp | 2 +- src/graphics/host_gpu/renderer/renderDraw.cpp | 2 +- .../shader/recompiler/ShaderRecompiler.cpp | 243 +- .../shader/recompiler/ShaderRecompiler.h | 2 +- .../spirv}/SpirvBuilder.cpp | 2 +- .../{emitter => backend/spirv}/SpirvBuilder.h | 0 .../recompiler/backend/spirv/SpirvEmitter.cpp | 396 + .../{emitter => backend/spirv}/SpirvEmitter.h | 4 +- .../spirv}/spirvEmitterAluOps.cpp | 2 +- .../spirv}/spirvEmitterAnalysis.cpp | 333 +- .../backend/spirv/spirvEmitterControlFlow.cpp | 52 + .../spirv}/spirvEmitterExport.cpp | 2 +- .../spirv}/spirvEmitterFlowOps.cpp | 2 +- .../spirv}/spirvEmitterImageHelpers.cpp | 2 +- .../spirv}/spirvEmitterImageOps.cpp | 2 +- .../spirv}/spirvEmitterInternal.h | 128 +- .../spirv}/spirvEmitterMemory.cpp | 19 +- .../spirv}/spirvEmitterModule.cpp | 17 +- .../backend/spirv/spirvEmitterValue.cpp | 541 + .../backend/spirv/spirvEmitterValueAlu.cpp | 677 + .../backend/spirv/spirvEmitterValueFlow.cpp | 557 + .../backend/spirv/spirvEmitterValueImage.cpp | 516 + .../backend/spirv/spirvEmitterValueMemory.cpp | 713 + .../spirv}/spirvEmitterValues.cpp | 5 +- .../recompiler/emitter/SpirvEmitter.cpp | 525 - .../emitter/spirvEmitterControlFlow.cpp | 1138 - .../{ => frontend}/cfg/ShaderCFG.cpp | 461 +- .../recompiler/{ => frontend}/cfg/ShaderCFG.h | 10 +- .../decode}/ExportOps.cpp | 2 +- .../decode}/ExportOps.h | 2 +- .../decode}/ImageOps.cpp | 2 +- .../decode}/ImageOps.h | 2 +- .../decode}/MemoryOps.cpp | 2 +- .../decode}/MemoryOps.h | 2 +- .../decode}/OpcodeTable.h | 2 +- .../decode}/ScalarAluOps.cpp | 4 +- .../decode}/ScalarAluOps.h | 2 +- .../decode}/ShaderDecoder.cpp | 13 +- .../decode}/ShaderDecoder.h | 45 +- .../decode}/VectorAluOps.cpp | 10 +- .../decode}/VectorAluOps.h | 2 +- .../frontend/translate/Attribute.cpp | 35 + .../recompiler/frontend/translate/Compare.cpp | 195 + .../recompiler/frontend/translate/Control.cpp | 260 + .../recompiler/frontend/translate/Convert.cpp | 158 + .../frontend/translate/Dispatch.cpp | 40 + .../recompiler/frontend/translate/Float.cpp | 212 + .../recompiler/frontend/translate/Integer.cpp | 630 + .../recompiler/frontend/translate/Memory.cpp | 648 + .../frontend/translate/Translate.cpp | 927 + .../recompiler/frontend/translate/Translate.h | 12 + .../frontend/translate/Translator.h | 140 + src/graphics/shader/recompiler/ir/Block.cpp | 84 + src/graphics/shader/recompiler/ir/Block.h | 63 + .../shader/recompiler/ir/IREmitter.cpp | 218 + src/graphics/shader/recompiler/ir/IREmitter.h | 83 + .../recompiler/ir/ReadLaneElimination.cpp | 323 - .../recompiler/ir/ReadLaneElimination.h | 20 - src/graphics/shader/recompiler/ir/Reg.h | 24 + .../shader/recompiler/ir/ResourceTracking.cpp | 464 - .../shader/recompiler/ir/ScalarProvenance.cpp | 963 - .../shader/recompiler/ir/ScalarProvenance.h | 20 - .../shader/recompiler/ir/ShaderIR.cpp | 41 +- src/graphics/shader/recompiler/ir/ShaderIR.h | 186 +- .../shader/recompiler/ir/ShaderIRInternal.h | 1 + .../shader/recompiler/ir/ShaderIRLog.cpp | 3 + .../shader/recompiler/ir/ShaderIROpcodes.inc | 342 - .../recompiler/ir/ShaderInfoCollection.cpp | 235 - .../shader/recompiler/ir/SrtPatcher.cpp | 138 - .../shader/recompiler/ir/SrtPatcher.h | 14 - .../shader/recompiler/ir/SrtWalker.cpp | 624 - src/graphics/shader/recompiler/ir/Type.cpp | 55 + src/graphics/shader/recompiler/ir/Type.h | 51 + src/graphics/shader/recompiler/ir/Value.cpp | 277 + src/graphics/shader/recompiler/ir/Value.h | 159 + .../shader/recompiler/ir/ValueProgram.cpp | 241 + .../shader/recompiler/ir/ValueProgram.h | 64 + .../shader/recompiler/ir/opcodes/Opcodes.h | 71 + .../ir/{ => opcodes}/ShaderIROpcodes.cpp | 1 + .../recompiler/ir/opcodes/ShaderIROpcodes.inc | 343 + .../recompiler/ir/opcodes/ValueOpcodes.cpp | 144 + .../recompiler/ir/opcodes/ValueOpcodes.h | 57 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 299 + .../ir/{ => passes}/BindingLayout.cpp | 131 +- .../ir/{ => passes}/BindingLayout.h | 0 .../ir/passes/ConstantPropagation.cpp | 612 + .../ir/passes/ConstantPropagation.h | 9 + .../ir/passes/DeadCodeElimination.cpp | 40 + .../ir/passes/DeadCodeElimination.h | 10 + .../ir/passes/ReadLaneElimination.cpp | 141 + .../ir/passes/ReadLaneElimination.h | 13 + .../{ => passes}/ResourceMaterialization.cpp | 82 +- .../ir/{ => passes}/ResourceMaterialization.h | 2 +- .../recompiler/ir/passes/ResourceTracking.cpp | 677 + .../ir/{ => passes}/ResourceTracking.h | 4 +- .../ir/passes/ShaderInfoCollection.cpp | 318 + .../ir/{ => passes}/ShaderInfoCollection.h | 0 .../shader/recompiler/ir/passes/SrtWalker.cpp | 951 + .../recompiler/ir/{ => passes}/SrtWalker.h | 5 +- .../recompiler/ir/passes/SsaRewrite.cpp | 377 + .../shader/recompiler/ir/passes/SsaRewrite.h | 9 + src/graphics/shader/rectListShader.cpp | 2 +- src/graphics/shader/shader.cpp | 6 +- src/graphics/shader/shaderStageRuntime.cpp | 2 +- tests/ResourceTrackingTests.cpp | 2467 +- tests/ScalarProvenanceTests.cpp | 1523 +- tests/ShaderRecompilerComputeTests.cpp | 35163 ++++++++-------- tests/ShaderStageRuntimeTests.cpp | 175 +- tests/shaderCfgTests.cpp | 14955 ++++--- 113 files changed, 40415 insertions(+), 32616 deletions(-) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/SpirvBuilder.cpp (98%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/SpirvBuilder.h (100%) create mode 100644 src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp rename src/graphics/shader/recompiler/{emitter => backend/spirv}/SpirvEmitter.h (84%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterAluOps.cpp (99%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterAnalysis.cpp (60%) create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterExport.cpp (99%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterFlowOps.cpp (99%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterImageHelpers.cpp (99%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterImageOps.cpp (99%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterInternal.h (94%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterMemory.cpp (99%) rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterModule.cpp (97%) create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAlu.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp create mode 100644 src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp rename src/graphics/shader/recompiler/{emitter => backend/spirv}/spirvEmitterValues.cpp (99%) delete mode 100644 src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp delete mode 100644 src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp rename src/graphics/shader/recompiler/{ => frontend}/cfg/ShaderCFG.cpp (80%) rename src/graphics/shader/recompiler/{ => frontend}/cfg/ShaderCFG.h (92%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ExportOps.cpp (95%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ExportOps.h (87%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ImageOps.cpp (99%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ImageOps.h (88%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/MemoryOps.cpp (99%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/MemoryOps.h (93%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/OpcodeTable.h (94%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ScalarAluOps.cpp (98%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ScalarAluOps.h (93%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ShaderDecoder.cpp (99%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/ShaderDecoder.h (92%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/VectorAluOps.cpp (99%) rename src/graphics/shader/recompiler/{decompiler => frontend/decode}/VectorAluOps.h (94%) create mode 100644 src/graphics/shader/recompiler/frontend/translate/Attribute.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Compare.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Control.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Convert.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Float.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Integer.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Memory.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Translate.cpp create mode 100644 src/graphics/shader/recompiler/frontend/translate/Translate.h create mode 100644 src/graphics/shader/recompiler/frontend/translate/Translator.h create mode 100644 src/graphics/shader/recompiler/ir/Block.cpp create mode 100644 src/graphics/shader/recompiler/ir/Block.h create mode 100644 src/graphics/shader/recompiler/ir/IREmitter.cpp create mode 100644 src/graphics/shader/recompiler/ir/IREmitter.h delete mode 100644 src/graphics/shader/recompiler/ir/ReadLaneElimination.cpp delete mode 100644 src/graphics/shader/recompiler/ir/ReadLaneElimination.h create mode 100644 src/graphics/shader/recompiler/ir/Reg.h delete mode 100644 src/graphics/shader/recompiler/ir/ResourceTracking.cpp delete mode 100644 src/graphics/shader/recompiler/ir/ScalarProvenance.cpp delete mode 100644 src/graphics/shader/recompiler/ir/ScalarProvenance.h delete mode 100644 src/graphics/shader/recompiler/ir/ShaderIROpcodes.inc delete mode 100644 src/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp delete mode 100644 src/graphics/shader/recompiler/ir/SrtPatcher.cpp delete mode 100644 src/graphics/shader/recompiler/ir/SrtPatcher.h delete mode 100644 src/graphics/shader/recompiler/ir/SrtWalker.cpp create mode 100644 src/graphics/shader/recompiler/ir/Type.cpp create mode 100644 src/graphics/shader/recompiler/ir/Type.h create mode 100644 src/graphics/shader/recompiler/ir/Value.cpp create mode 100644 src/graphics/shader/recompiler/ir/Value.h create mode 100644 src/graphics/shader/recompiler/ir/ValueProgram.cpp create mode 100644 src/graphics/shader/recompiler/ir/ValueProgram.h create mode 100644 src/graphics/shader/recompiler/ir/opcodes/Opcodes.h rename src/graphics/shader/recompiler/ir/{ => opcodes}/ShaderIROpcodes.cpp (99%) create mode 100644 src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc create mode 100644 src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp create mode 100644 src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h create mode 100644 src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc rename src/graphics/shader/recompiler/ir/{ => passes}/BindingLayout.cpp (70%) rename src/graphics/shader/recompiler/ir/{ => passes}/BindingLayout.h (100%) create mode 100644 src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp create mode 100644 src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h create mode 100644 src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp create mode 100644 src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.h create mode 100644 src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp create mode 100644 src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h rename src/graphics/shader/recompiler/ir/{ => passes}/ResourceMaterialization.cpp (89%) rename src/graphics/shader/recompiler/ir/{ => passes}/ResourceMaterialization.h (96%) create mode 100644 src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp rename src/graphics/shader/recompiler/ir/{ => passes}/ResourceTracking.h (72%) create mode 100644 src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp rename src/graphics/shader/recompiler/ir/{ => passes}/ShaderInfoCollection.h (100%) create mode 100644 src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp rename src/graphics/shader/recompiler/ir/{ => passes}/SrtWalker.h (96%) create mode 100644 src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp create mode 100644 src/graphics/shader/recompiler/ir/passes/SsaRewrite.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e675d952..60aeae8b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,14 +203,21 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS "${KYTY_SOURCE_DIR}/graphics/shader/*.h" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.h" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.h" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.h" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.h" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/cfg/*.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/cfg/*.h" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/decode/*.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/decode/*.h" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/translate/*.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/translate/*.h" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/backend/spirv/*.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/backend/spirv/*.h" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.h" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.h" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.inc" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/*.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/*.h" "${KYTY_SOURCE_DIR}/graphics/presentation/*.cpp" "${KYTY_SOURCE_DIR}/graphics/presentation/*.h" "${KYTY_SOURCE_DIR}/graphics/presentation/window/*.cpp" @@ -367,9 +374,8 @@ add_kyty_full_emulator_test(shader_cfg_tests "${KYTY_TESTS_DIR}/shaderCfgTests.c add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/ScalarProvenanceTests.cpp" "${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ReadLaneElimination.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp" ) target_link_libraries(scalar_provenance_tests fmt::fmt) target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers}) @@ -413,9 +419,8 @@ add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL "${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp" "${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/shaderStageRuntime.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp" ) target_link_libraries(shader_stage_runtime_tests fmt::fmt) target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers}) @@ -424,13 +429,11 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp" "${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp" "${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtPatcher.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceTracking.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/BindingLayout.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp" + "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/BindingLayout.cpp" ) target_link_libraries(resource_tracking_tests fmt::fmt) target_include_directories(resource_tracking_tests PRIVATE ${inc_headers}) diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index aedb5f5ab..d3e695cdf 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -23,9 +23,9 @@ #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vma.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/shader/recompiler/ir/BindingLayout.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h index a68b295a5..27643fe20 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h +++ b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h @@ -2,7 +2,7 @@ #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_ #include "graphics/host_gpu/graphicContext.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp b/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp index f96b8cc88..5836081bf 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp @@ -1,6 +1,6 @@ #include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" -#include "graphics/shader/recompiler/emitter/SpirvEmitter.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" namespace Libs::Graphics { diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 6aa624407..4d3342a16 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -19,8 +19,8 @@ #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include "kernel/eventQueue.h" #include "kernel/pthread.h" diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index abcc2024e..cdb9cc010 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -22,8 +22,8 @@ #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/host_gpu/vulkanCommon.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include "kernel/eventQueue.h" #include "kernel/memory.h" diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 36071c826..398501ffa 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -2,18 +2,21 @@ #include "common/assert.h" #include "common/logging/log.h" -#include "graphics/shader/recompiler/cfg/ShaderCFG.h" -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" -#include "graphics/shader/recompiler/emitter/SpirvEmitter.h" -#include "graphics/shader/recompiler/ir/BindingLayout.h" -#include "graphics/shader/recompiler/ir/ReadLaneElimination.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" -#include "graphics/shader/recompiler/ir/ResourceTracking.h" -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" +#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/translate/Translate.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/SrtPatcher.h" -#include "graphics/shader/recompiler/ir/SrtWalker.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" +#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" +#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" +#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" #include #include @@ -43,7 +46,7 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) { std::string dump = "CFG:\n"; dump += CFG::GraphToString(cfg); dump += "\nIR:\n"; - dump += IR::ProgramToString(ir); + dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir); return dump; } @@ -56,10 +59,22 @@ const char* StageName(ShaderType stage) { } } -std::string FormatCfgFailure(const CFG::Graph& cfg, const CompileOptions& options, - const std::string& reason) { - const auto block_id = cfg.failure_block != UINT32_MAX ? cfg.failure_block : cfg.entry_block; - return CFG::FormatBlockDiagnostic(cfg, block_id, StageName(options.stage), reason); +void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase, + const std::string& reason) { + const auto* block = cfg.FindBlock(cfg.failure_block); + const auto start = block != nullptr ? block->start_pc : UINT32_MAX; + const auto end = block != nullptr ? block->end_pc : UINT32_MAX; + const auto predecessors = block != nullptr ? block->predecessors.size() : 0u; + const auto successors = block != nullptr ? block->successors.size() : 0u; + LOGF("%s CFG dispatcher fallback: stage=%s hash=0x%016" PRIx64 + " phase=%s failure=%s block=%" PRIu32 " pc=0x%08" PRIx32 "..0x%08" PRIx32 " preds=%" PRIu64 + " succs=%" PRIu64 " blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64 + " reason=%s\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, phase, + CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end, + static_cast(predecessors), static_cast(successors), + static_cast(cfg.blocks.size()), static_cast(cfg.natural_loops.size()), + static_cast(cfg.back_edges.size()), reason.c_str()); } bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) { @@ -94,21 +109,31 @@ bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) { case IR::Opcode::FlatStoreByte: case IR::Opcode::FlatStoreShort: case IR::Opcode::FlatStoreDword: + case IR::Opcode::DsReadUbyte: + case IR::Opcode::DsReadSbyte: + case IR::Opcode::DsReadUshort: + case IR::Opcode::DsReadSshort: + case IR::Opcode::DsReadB32: case IR::Opcode::DsMinF32: case IR::Opcode::DsMaxF32: case IR::Opcode::DsWriteByte: case IR::Opcode::DsWriteShort: case IR::Opcode::DsWriteB32: case IR::Opcode::DsWriteAddtidB32: + case IR::Opcode::DsReadAddtidB32: case IR::Opcode::DsAppend: case IR::Opcode::DsConsume: + case IR::Opcode::ImageLoad: case IR::Opcode::ImageStore: + case IR::Opcode::ImageSample: + case IR::Opcode::ImageGather4: case IR::Opcode::Export: return true; default: return false; } } -bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* reason) { +bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, uint32_t* failure_block, + std::string* reason) { for (const auto& block: ir.blocks) { if (!block.terminator.loop_header) { continue; @@ -122,6 +147,9 @@ bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* "lowering emits internal control flow: {}", block.id, IR::InstructionToString(inst).c_str()); } + if (failure_block != nullptr) { + *failure_block = block.id; + } return true; } } @@ -697,27 +725,25 @@ bool TryRecompile(std::span code, const CompileOptions& options, bool dispatcher_fallback = false; std::string dispatcher_reason; if (cfg.irreducible) { - dispatcher_fallback = true; - dispatcher_reason = cfg.unsupported_reason; - const auto diagnostic = FormatCfgFailure(cfg, options, cfg.unsupported_reason); - LOGF("%s irreducible CFG detected: %s\n", GetDumpLabel(options), diagnostic.c_str()); + dispatcher_fallback = true; + dispatcher_reason = cfg.unsupported_reason; + LogDispatcherFallback(options, cfg, "build", dispatcher_reason); } else { std::string structure_error; const auto unstructured_cfg = cfg; LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); if (!CFG::Structurize(cfg, &structure_error)) { - const auto diagnostic = FormatCfgFailure(cfg, options, structure_error); - LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str()); dispatcher_fallback = true; dispatcher_reason = structure_error; const auto failure_kind = cfg.failure_kind; const auto failure_block = cfg.failure_block; - cfg = unstructured_cfg; - cfg.unsupported = true; - cfg.failure_kind = failure_kind; - cfg.failure_block = failure_block; - cfg.unsupported_reason = structure_error; + LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason); + cfg = unstructured_cfg; + cfg.unsupported = true; + cfg.failure_kind = failure_kind; + cfg.failure_block = failure_block; + cfg.unsupported_reason = structure_error; } else { LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options), static_cast(cfg.blocks.size())); @@ -732,13 +758,84 @@ bool TryRecompile(std::span code, const CompileOptions& options, IR::Program ir; LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); - if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, ir, error)) { + const auto lower_program = [&](IR::Program& output) { + output = {}; + if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) { + return false; + } + output.lane_mask_mode = options.lane_mask_mode; + output.shader_hash = options.shader_hash; + output.user_data_base = options.user_data_base; + output.user_data_count = options.user_data_count; + return true; + }; + if (!lower_program(ir)) { return false; } - ir.lane_mask_mode = options.lane_mask_mode; - ir.shader_hash = options.shader_hash; - ir.user_data_base = options.user_data_base; - ir.user_data_count = options.user_data_count; + + if (!dispatcher_fallback) { + const auto select_dispatcher = [&](uint32_t failure_block, const std::string& reason, + const CFG::Graph& diagnostic_cfg) { + dispatcher_fallback = true; + dispatcher_reason = reason; + cfg.failure_kind = CFG::FailureKind::StructuredControlFlow; + cfg.failure_block = failure_block; + cfg.unsupported_reason = reason; + ir.cfg_failure_kind = CFG::FailureKind::StructuredControlFlow; + ir.fallback_reason = reason; + LogDispatcherFallback(options, diagnostic_cfg, "emit", reason); + }; + const auto retry_budget = cfg.natural_loops.size(); + for (size_t retry = 0;; retry++) { + std::string emitter_reason; + uint32_t failure_block = UINT32_MAX; + if (!NeedsDispatcherForStructuredLoopHeader(ir, &failure_block, &emitter_reason)) { + break; + } + if (retry >= retry_budget) { + select_dispatcher(failure_block, + emitter_reason + "; no-clone loop-header isolation exhausted", + cfg); + break; + } + + const auto* source_block = cfg.FindBlock(failure_block); + const auto source_pc = source_block != nullptr ? source_block->start_pc : UINT32_MAX; + const auto previous_cfg = cfg; + const auto blocks_before = cfg.blocks.size(); + std::string isolation_error; + if (!CFG::IsolateLoopHeader(cfg, failure_block, &isolation_error)) { + const auto failed_cfg = cfg; + cfg = previous_cfg; + select_dispatcher(failure_block, + fmt::format("{}; no-clone loop-header isolation failed: {}", + emitter_reason, isolation_error), + failed_cfg); + break; + } + + IR::Program isolated_ir; + if (!lower_program(isolated_ir)) { + const auto lower_error = error != nullptr ? *error : "IR lowering failed"; + if (error != nullptr) { + error->clear(); + } + cfg = previous_cfg; + select_dispatcher(failure_block, + fmt::format("{}; isolated loop-header lowering failed: {}", + emitter_reason, lower_error), + cfg); + break; + } + ir = std::move(isolated_ir); + LOGF("%s CFG isolated semantic loop header: stage=%s hash=0x%016" PRIx64 + " source_block=%" PRIu32 " pc=0x%08" PRIx32 " blocks=%" PRIu64 "->%" PRIu64 + " reason=%s\n", + GetDumpLabel(options), StageName(options.stage), options.shader_hash, + failure_block, source_pc, static_cast(blocks_before), + static_cast(cfg.blocks.size()), emitter_reason.c_str()); + } + } LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash, @@ -757,9 +854,41 @@ bool TryRecompile(std::span code, const CompileOptions& options, rewritten); } } - if (!IR::BuildScalarProvenance(ir, error)) { + ir.dispatcher_fallback = dispatcher_fallback; + if (!dispatcher_reason.empty()) { + ir.fallback_reason = dispatcher_reason; + } + + ShaderVertexInputInfo default_vertex {}; + ShaderPixelInputInfo default_pixel {}; + ShaderComputeInputInfo default_compute {}; + const auto* vertex = + options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex; + const auto* pixel = + options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel; + const auto* compute = + options.compute_input_info != nullptr ? options.compute_input_info : &default_compute; + ir.values = std::make_shared(); + if (!Frontend::TranslateProgram(ir, *ir.values, vertex, pixel, compute, error)) { return false; } + ir.blocks.clear(); + IR::RewriteToSsa(ir.values->blocks); + IR::ConstantPropagationPass(ir.values->blocks); + for (auto& info: ir.values->block_info) { + info.condition = info.condition.Resolve(); + info.indirect_target = info.indirect_target.Resolve(); + } + IR::RemoveIdentities(ir.values->blocks); + IR::EliminateDeadCode(ir.values->blocks); + const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size); + if (read_lane_stats.rewritten_reads != 0) { + LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options), + read_lane_stats.rewritten_reads); + IR::ConstantPropagationPass(ir.values->blocks); + IR::RemoveIdentities(ir.values->blocks); + IR::EliminateDeadCode(ir.values->blocks); + } std::string srt_error; if (!IR::BuildSrtPlan(ir, &srt_error)) { LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str()); @@ -768,29 +897,13 @@ bool TryRecompile(std::span code, const CompileOptions& options, } return false; } - ir.dispatcher_fallback = dispatcher_fallback; - if (!dispatcher_reason.empty()) { - ir.fallback_reason = dispatcher_reason; - } - - if (!IR::PatchSrtReads(ir, error) || !IR::TrackResources(ir, error)) { + IR::EliminateDeadCode(ir.values->blocks); + if (!IR::TrackResources(ir, error)) { return false; } if (options.stage == ShaderType::Vertex) { ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr; } - if (!dispatcher_fallback) { - std::string emitter_reason; - if (NeedsDispatcherForStructuredLoopHeader(ir, &emitter_reason)) { - dispatcher_fallback = true; - dispatcher_reason = emitter_reason; - ir.dispatcher_fallback = true; - ir.cfg_failure_kind = CFG::FailureKind::StructuredControlFlow; - ir.fallback_reason = emitter_reason; - LOGF("%s structured CFG emitter fallback: %s\n", GetDumpLabel(options), - emitter_reason.c_str()); - } - } IR::ResourceSnapshot resources; if (options.resource_snapshot != nullptr) { @@ -822,16 +935,10 @@ bool TryRecompile(std::span code, const CompileOptions& options, return false; } - ShaderVertexInputInfo default_vertex {}; - ShaderPixelInputInfo default_pixel {}; - ShaderComputeInputInfo default_compute {}; - IR::ShaderInfoOptions info_options; - info_options.vertex = - options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex; - info_options.pixel = - options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel; - info_options.compute = - options.compute_input_info != nullptr ? options.compute_input_info : &default_compute; + IR::ShaderInfoOptions info_options; + info_options.vertex = vertex; + info_options.pixel = pixel; + info_options.compute = compute; if (!IR::CollectShaderInfo(ir, info_options, error)) { return false; } @@ -841,11 +948,6 @@ bool TryRecompile(std::span code, const CompileOptions& options, if (!IR::AllocateBindings(ir, layout_options, error)) { return false; } - const auto read_lane_stats = IR::EliminateReadLane(ir); - if (read_lane_stats.rewritten_reads != 0) { - LOGF("%s read-lane elimination: reads=%" PRIu32 " shadow_writes=%" PRIu32 "\n", - GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.shadow_writes); - } std::string ir_dump; if (options.dump_ir) { ir_dump = MakeIrDump(cfg, ir); @@ -859,7 +961,8 @@ bool TryRecompile(std::span code, const CompileOptions& options, LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info, - options.compute_input_info, spirv, &emit_error)) { + options.compute_input_info, spirv, &emit_error, options.dump_ir)) { + LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str()); if (dispatcher_fallback && error != nullptr) { *error = fmt::format("dispatcher fallback failed after {}: {}", dispatcher_reason.c_str(), emit_error.c_str()); @@ -873,10 +976,6 @@ bool TryRecompile(std::span code, const CompileOptions& options, " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash, static_cast(spirv.size()), phase_ms()); - if (dispatcher_fallback) { - LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str()); - } - result.spirv = std::move(spirv); result.program = std::move(ir); result.resources = std::move(resources); diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index bf7d1fb4d..072f56148 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -3,7 +3,7 @@ #include "common/common.h" #include "common/stringUtils.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include diff --git a/src/graphics/shader/recompiler/emitter/SpirvBuilder.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp similarity index 98% rename from src/graphics/shader/recompiler/emitter/SpirvBuilder.cpp rename to src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp index c6ae631ff..e9b4d48b9 100644 --- a/src/graphics/shader/recompiler/emitter/SpirvBuilder.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/SpirvBuilder.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include #include diff --git a/src/graphics/shader/recompiler/emitter/SpirvBuilder.h b/src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h similarity index 100% rename from src/graphics/shader/recompiler/emitter/SpirvBuilder.h rename to src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp new file mode 100644 index 000000000..15188e167 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -0,0 +1,396 @@ +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" + +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv { + +namespace { + +bool Fail(std::string* error, const std::string& message) { + if (error != nullptr) { + *error = message; + } + return false; +} + +bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) { + using Kind = IR::DescriptorBindingKind; + using Dim = Decoder::ImageDimension; + if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) { + const bool integer = image.kind == IR::ResourceKind::ImageUint; + switch (image.dimension) { + case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true; + case Dim::Dim1DArray: + kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray; + return true; + case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true; + case Dim::Dim2DMsaa: + kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa; + return true; + case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true; + case Dim::Dim2DArray: + kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray; + return true; + case Dim::Dim2DMsaaArray: + kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray; + return true; + case Dim::Unknown: return false; + } + } + const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint; + if (image.kind != IR::ResourceKind::StorageImage && !uint_image) { + return false; + } + switch (image.dimension) { + case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true; + case Dim::Dim1DArray: + kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray; + return true; + case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true; + case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true; + case Dim::Dim2DArray: + kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray; + return true; + case Dim::Dim2DMsaa: + case Dim::Dim2DMsaaArray: return false; + case Dim::Unknown: return false; + } + return false; +} + +bool ValidateNativeProgram(const IR::Program& program, std::string* error) { + using Kind = IR::DescriptorBindingKind; + constexpr auto KindCount = static_cast(Kind::Count); + std::array, KindCount> expected; + std::array present {}; + const auto Dense = [](size_t size) { + std::vector values(size); + for (uint32_t i = 0; i < values.size(); i++) { + values[i] = i; + } + return values; + }; + auto Expect = [&](Kind kind, std::vector resources = {}) { + const auto index = static_cast(kind); + present[index] = true; + expected[index] = std::move(resources); + }; + if (!program.info.buffers.empty()) { + Expect(Kind::Buffers, Dense(program.info.buffers.size())); + } + for (uint32_t i = 0; i < program.info.images.size(); i++) { + Kind kind; + if (!ImageBinding(program.info.images[i], kind)) { + return Fail(error, "native shader plan has an invalid image class"); + } + present[static_cast(kind)] = true; + expected[static_cast(kind)].push_back(i); + } + if (!program.info.samplers.empty()) { + Expect(Kind::Samplers, Dense(program.info.samplers.size())); + } + const auto uses_gds = program.values != nullptr && + std::ranges::any_of(program.values->blocks, [](const IR::Block* block) { + return std::ranges::any_of(*block, [](const IR::Inst& inst) { + return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource; + }); + }); + if (uses_gds) { + Expect(Kind::Gds); + } + if (!program.info.addresses.empty()) { + Expect(Kind::AddressMemory, Dense(program.info.addresses.size())); + } + if (program.values != nullptr && !program.values->srt_reads.empty()) { + Expect(Kind::FlattenedSrt); + } + if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) { + Expect(Kind::UserData); + } + + std::array seen {}; + for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) { + const auto& binding = program.bindings.descriptors[i]; + const auto kind = static_cast(binding.kind); + if (kind >= KindCount || seen[kind] || !present[kind] || + binding.resources != expected[kind]) { + return Fail(error, "native descriptor groups do not match shader topology"); + } + for (uint32_t j = 0; j < i; j++) { + if (program.bindings.descriptors[j].binding == binding.binding) { + return Fail(error, "native descriptor binding numbers are not unique"); + } + } + seen[kind] = true; + } + for (size_t i = 0; i < KindCount; i++) { + if (present[i] != seen[i]) { + return Fail(error, "native shader plan is missing a required descriptor group"); + } + } + const auto has_user_storage = present[static_cast(Kind::UserData)]; + if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() || + program.bindings.buffer_offset_count != program.info.buffers.size() || + (!has_user_storage && + program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) || + (has_user_storage && program.bindings.push_constant_size != 0) || + !std::is_sorted(program.bindings.user_data_registers.begin(), + program.bindings.user_data_registers.end()) || + std::adjacent_find(program.bindings.user_data_registers.begin(), + program.bindings.user_data_registers.end()) != + program.bindings.user_data_registers.end()) { + return Fail(error, "native user-data layout is inconsistent"); + } + + if (program.values == nullptr) { + return Fail(error, "native shader plan has no typed SSA"); + } + const auto planning_only_handle = [&](const IR::Inst& handle) { + return !handle.Uses().empty() && + std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) { + const auto op = use.user->GetOpcode(); + if (op != IR::ValueOpcode::LoadAddressU32 && + op != IR::ValueOpcode::ReadConstBuffer) { + return false; + } + const auto index = use.user->Flags().index; + return index < program.values->memory_info.size() && + program.values->memory_info[index].planning_only; + }); + }; + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + const auto dense = inst.Flags(); + switch (inst.GetOpcode()) { + case IR::ValueOpcode::GetBufferResource: + if (planning_only_handle(inst)) { + break; + } + if (dense >= program.info.buffers.size()) { + return Fail(error, "typed buffer handle has an invalid dense resource"); + } + break; + case IR::ValueOpcode::GetAddressResource: + if (planning_only_handle(inst)) { + break; + } + if (dense >= program.info.addresses.size()) { + return Fail(error, "typed address handle has an invalid dense resource"); + } + break; + case IR::ValueOpcode::GetImageResource: + if (dense >= program.info.images.size()) { + return Fail(error, "typed image handle has an invalid dense resource"); + } + break; + case IR::ValueOpcode::GetSamplerResource: + if (dense >= program.info.samplers.size()) { + return Fail(error, "typed sampler handle has an invalid dense resource"); + } + break; + case IR::ValueOpcode::ReadConst: { + const auto slot = inst.Arg(1).Resolve(); + if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 || + slot.U32() >= program.values->srt_reads.size()) { + return Fail(error, "flattened SRT read has an invalid dense slot"); + } + break; + } + default: break; + } + } + } + return true; +} + +} // namespace + +static constexpr size_t InitialEmitterVectorReserve = 4096; + +static void ReportReserveExceeded(const IR::Program& program, const char* vector_name, + size_t size) { + if (size <= InitialEmitterVectorReserve) { + return; + } + std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64 + " stage=%u vector=%s size=%zu reserve=%zu\n", + program.shader_hash, static_cast(program.stage), vector_name, size, + InitialEmitterVectorReserve); +} + +void CollectValueRegisterMirrors(const IR::ValueProgram& program, + std::vector& registers) { + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + switch (inst.GetOpcode()) { + case IR::ValueOpcode::SetScalarRegister: + case IR::ValueOpcode::SetThreadBitScalarRegister: + if (block != program.blocks.front()) { + Emitter::CollectRegister( + registers, + {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())}); + } + break; + case IR::ValueOpcode::SetVectorRegister: + Emitter::CollectRegister( + registers, + {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())}); + break; + case IR::ValueOpcode::SetM0: + if (block != program.blocks.front()) { + Emitter::CollectRegister(registers, {IR::RegisterFile::M0, 0}); + } + break; + case IR::ValueOpcode::SetExecLo: + case IR::ValueOpcode::SetExecHi: + if (block != program.blocks.front()) { + Emitter::CollectRegister( + registers, {IR::RegisterFile::Exec, + inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u}); + } + break; + case IR::ValueOpcode::SetVccLo: + case IR::ValueOpcode::SetVccHi: + if (block != program.blocks.front()) { + Emitter::CollectRegister( + registers, {IR::RegisterFile::Vcc, + inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u}); + } + break; + default: break; + } + } + } +} + +bool ValueProgramHasOpcode(const IR::ValueProgram& program, IR::ValueOpcode opcode) { + return std::ranges::any_of(program.blocks, [opcode](const IR::Block* block) { + return std::ranges::any_of( + *block, [opcode](const IR::Inst& inst) { return inst.GetOpcode() == opcode; }); + }); +} + +bool ProgramRequiresExactSubgroupSize(const IR::Program& program) { + if (program.values == nullptr) { + return false; + } + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + switch (inst.GetOpcode()) { + case IR::ValueOpcode::Ballot: + case IR::ValueOpcode::DppMoveU32: + case IR::ValueOpcode::DppUpdateU32: + case IR::ValueOpcode::WqmMask: + case IR::ValueOpcode::ReadFirstLane: + case IR::ValueOpcode::ReadLane: + case IR::ValueOpcode::WriteLane: + case IR::ValueOpcode::Permlane16U32: + case IR::ValueOpcode::SwizzleU32: + case IR::ValueOpcode::DataAppend: + case IR::ValueOpcode::DataConsume: + case IR::ValueOpcode::GdsDataAppend: + case IR::ValueOpcode::GdsDataConsume: return true; + default: break; + } + } + } + return false; +} + +bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, + const ShaderVertexInputInfo* vertex_input_info, + const ShaderPixelInputInfo* pixel_input_info, + const ShaderComputeInputInfo* compute_input_info, std::vector& spirv, + std::string* error, bool preserve_debug_values) { + using namespace Emitter; + + if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && + program.stage != ShaderType::Pixel) { + SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders"); + return false; + } + if (!program.srt_plan_complete || !program.resource_tracking_complete || + !program.shader_info_complete || !program.binding_layout_complete) { + SetError(error, "SPIR-V emitter requires a fully planned native shader program"); + return false; + } + if (!IR::ValidateResourceSnapshot(program, resources, error)) { + return false; + } + if (!IR::ValidateResourceSpecialization(program, resources, error)) { + return false; + } + if (!ValidateNativeProgram(program, error)) { + return false; + } + if (program.values == nullptr) { + SetError(error, "SPIR-V emitter requires planned typed SSA"); + return false; + } + ShaderVertexInputInfo default_vertex {}; + ShaderPixelInputInfo default_pixel {}; + ShaderComputeInputInfo default_compute {}; + const auto* vertex = vertex_input_info != nullptr ? vertex_input_info : &default_vertex; + const auto* pixel = pixel_input_info != nullptr ? pixel_input_info : &default_pixel; + const auto* compute = compute_input_info != nullptr ? compute_input_info : &default_compute; + (void)preserve_debug_values; + const auto& value_program = *program.values; + if (!IR::ValidateValueProgram(value_program, true, error)) { + return false; + } + EmitterState state(program, resources); + state.vertex_input_info = vertex; + state.pixel_input_info = pixel; + state.compute_input_info = compute; + state.stage = program.stage; + state.wave_size = program.wave_size; + state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; + state.exact_subgroup_operations = + !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program); + state.registers.reserve(InitialEmitterVectorReserve); + state.inputs.reserve(InitialEmitterVectorReserve); + state.outputs.reserve(InitialEmitterVectorReserve); + state.interface_variables.reserve(InitialEmitterVectorReserve); + state.reachable_blocks.reserve(InitialEmitterVectorReserve); + CollectValueRegisterMirrors(value_program, state.registers); + CopyProgramInputsAndOutputs(state, program); + state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program) || + (!state.per_invocation_masks && + ValueProgramHasOpcode(value_program, IR::ValueOpcode::Ballot)); + state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program); + state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program); + state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program); + state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program); + state.needs_function_lds = ProgramNeedsFunctionLds(program); + state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program); + AllocateInputVariables(state); + AllocateOutputVariables(state); + AllocateDescriptorVariables(state); + EmitHeaderAndTypes(state); + AllocateRegisterVariables(state); + if (!EmitValueProgram(state, value_program, error)) { + return false; + } + + ReportReserveExceeded(program, "registers", state.registers.size()); + ReportReserveExceeded(program, "inputs", state.inputs.size()); + ReportReserveExceeded(program, "outputs", state.outputs.size()); + ReportReserveExceeded(program, "interface_variables", state.interface_variables.size()); + ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size()); + + auto binary = state.builder.Build(); + if (binary.empty()) { + SetError(error, "SPIR-V builder returned an empty module"); + return false; + } + spirv = std::move(binary); + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/emitter/SpirvEmitter.h b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h similarity index 84% rename from src/graphics/shader/recompiler/emitter/SpirvEmitter.h rename to src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h index 0006b9bf3..c7f20a93b 100644 --- a/src/graphics/shader/recompiler/emitter/SpirvEmitter.h +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h @@ -3,7 +3,7 @@ #include "common/common.h" #include "common/stringUtils.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include @@ -15,7 +15,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc const ShaderVertexInputInfo* vertex_input_info, const ShaderPixelInputInfo* pixel_input_info, const ShaderComputeInputInfo* compute_input_info, std::vector& spirv, - std::string* error); + std::string* error, bool preserve_debug_values = false); } // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterAluOps.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterAluOps.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp index 4ecf53ca1..81216d32d 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterAluOps.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluOps.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp similarity index 60% rename from src/graphics/shader/recompiler/emitter/spirvEmitterAnalysis.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 48611a2f7..d2fde6a88 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -1,7 +1,8 @@ #include "common/assert.h" #include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/shader/recompiler/emitter/SpirvEmitter.h" -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { @@ -75,168 +76,6 @@ bool IsSccOperand(const IR::Operand& operand) { return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc; } -void CollectMaskStateRegisters(std::vector& registers) { - CollectRegister(registers, {IR::RegisterFile::Exec, 0}); - CollectRegister(registers, {IR::RegisterFile::Exec, 1}); - CollectRegister(registers, {IR::RegisterFile::Vcc, 0}); - CollectRegister(registers, {IR::RegisterFile::Vcc, 1}); - CollectRegister(registers, SccRegister()); -} - -void CollectSequentialRegisters(std::vector& registers, const IR::Operand& base, - uint32_t count) { - if (base.kind != IR::OperandKind::Register) { - return; - } - for (uint32_t i = 0; i < count; i++) { - auto reg = base.reg; - reg.index += i; - CollectRegister(registers, reg); - } -} - -uint32_t MaxCollectedVectorRegisterEnd(const std::vector& registers) { - uint32_t max_end = 0; - for (const auto& binding: registers) { - if (binding.reg.file == IR::RegisterFile::Vector) { - max_end = std::max(max_end, binding.reg.index + 1u); - } - } - return std::min(max_end, 256u); -} - -void CollectMoveRelSourceRegisters(const IR::Program& program, - std::vector& registers) { - const auto max_vector_end = MaxCollectedVectorRegisterEnd(registers); - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op != IR::Opcode::MoveRelSourceU32 || - inst.src[0].kind != IR::OperandKind::Register || - inst.src[0].reg.file != IR::RegisterFile::Vector) { - continue; - } - const auto base_index = inst.src[0].reg.index; - const auto limit = std::min(std::max(max_vector_end, base_index + 1u), 256u); - CollectSequentialRegisters(registers, inst.src[0], limit - base_index); - } - } -} - -bool IsPairDwordOpcode(IR::Opcode op) { - switch (op) { - case IR::Opcode::MoveU64: - case IR::Opcode::WqmB64: - case IR::Opcode::SaveexecB64: - case IR::Opcode::BitwiseAndU64: - case IR::Opcode::BitwiseAndNotU64: - case IR::Opcode::BitwiseOrU64: - case IR::Opcode::BitwiseOrNotU64: - case IR::Opcode::BitwiseXorU64: - case IR::Opcode::BitwiseNandU64: - case IR::Opcode::BitwiseNorU64: - case IR::Opcode::BitwiseXnorU64: - case IR::Opcode::BitwiseNotU64: - case IR::Opcode::BitFieldMaskU64: - case IR::Opcode::BitFieldExtractU64: - case IR::Opcode::BitReplicateB64B32: - case IR::Opcode::ShiftLeftLogicalU64: - case IR::Opcode::ShiftRightLogicalU64: - case IR::Opcode::SelectU64: return true; - default: return false; - } -} - -uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) { - switch (op) { - case IR::Opcode::BitFieldMaskU64: return 0; - case IR::Opcode::BitReplicateB64B32: return 0; - case IR::Opcode::BitFieldExtractU64: return 1; - case IR::Opcode::ShiftLeftLogicalU64: - case IR::Opcode::ShiftRightLogicalU64: - case IR::Opcode::BitwiseNotU64: - case IR::Opcode::SaveexecB64: - case IR::Opcode::WqmB64: - case IR::Opcode::MoveU64: return std::min(src_count, 1u); - default: return src_count; - } -} - -void CollectRegisters(const IR::Program& program, std::vector& registers) { - CollectMaskStateRegisters(registers); - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.dst.kind == IR::OperandKind::Register) { - CollectRegister(registers, inst.dst.reg); - } - if (inst.dst2.kind == IR::OperandKind::Register) { - CollectRegister(registers, inst.dst2.reg); - } - for (uint32_t i = 0; i < inst.src_count; i++) { - if (inst.src[i].kind == IR::OperandKind::Register) { - CollectRegister(registers, inst.src[i].reg); - } - } - if (IsPairDwordOpcode(inst.op)) { - CollectSequentialRegisters(registers, inst.dst, 2); - const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u; - const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count); - for (uint32_t i = first_pair_src; i < pair_src_count; i++) { - CollectSequentialRegisters(registers, inst.src[i], 2); - } - } - if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) { - CollectSequentialRegisters(registers, inst.src[0], 2); - } - if (IR::IsCompare64Opcode(inst.op)) { - CollectSequentialRegisters(registers, inst.src[0], 2); - CollectSequentialRegisters(registers, inst.src[1], 2); - } - if (IR::IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file != IR::RegisterFile::Scc) { - CollectSequentialRegisters(registers, inst.dst, 2); - } - if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) { - CollectSequentialRegisters(registers, inst.dst2, 2); - } - if (inst.op == IR::Opcode::UMadU64U32) { - CollectSequentialRegisters(registers, inst.dst, 2); - CollectSequentialRegisters(registers, inst.dst2, 2); - CollectSequentialRegisters(registers, inst.src[2], 2); - } - if (inst.op == IR::Opcode::SBufferLoadDword && - inst.memory.kind == IR::ResourceKind::ScalarBuffer) { - CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u}); - } - if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) && - inst.memory.image_address_components > 1u) { - CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords); - CollectSequentialRegisters(registers, inst.src[0], - inst.memory.image_address_components); - } - if (inst.op == IR::Opcode::ImageGetLod) { - CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords); - CollectSequentialRegisters(registers, inst.src[0], - inst.memory.image_address_components); - } - if (inst.op == IR::Opcode::ImageLoad) { - CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords); - CollectSequentialRegisters(registers, inst.src[0], - inst.memory.image_address_components); - } - if (inst.op == IR::Opcode::ImageStore) { - CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords); - CollectSequentialRegisters(registers, inst.src[1], - inst.memory.image_address_components); - } - if (inst.memory.kind == IR::ResourceKind::StorageImageUint) { - CollectSequentialRegisters(registers, inst.src[1], - inst.memory.image_address_components); - } - } - } - CollectMoveRelSourceRegisters(program, registers); -} - bool HasOutput(const std::vector& outputs, IR::StageOutputKind kind, uint32_t index) { return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) { @@ -276,9 +115,12 @@ uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo } bool ProgramNeedsComputeDerivatives(const IR::Program& program) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == IR::Opcode::ImageGetLod) { + if (program.values == nullptr) { + return false; + } + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() == IR::ValueOpcode::ImageQueryLod) { return true; } } @@ -287,9 +129,12 @@ bool ProgramNeedsComputeDerivatives(const IR::Program& program) { } bool ProgramNeedsImageGatherExtended(const IR::Program& program) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == IR::Opcode::ImageGather4) { + if (program.values == nullptr) { + return false; + } + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() == IR::ValueOpcode::ImageGatherRaw) { return true; } } @@ -297,34 +142,43 @@ bool ProgramNeedsImageGatherExtended(const IR::Program& program) { return false; } -bool IsLdsOpcode(IR::Opcode op) { - switch (op) { - case IR::Opcode::DsReadUbyte: - case IR::Opcode::DsReadSbyte: - case IR::Opcode::DsReadUshort: - case IR::Opcode::DsReadSshort: - case IR::Opcode::DsReadB32: - case IR::Opcode::DsWriteByte: - case IR::Opcode::DsWriteShort: - case IR::Opcode::DsWriteB32: - case IR::Opcode::DsMinF32: - case IR::Opcode::DsMaxF32: - case IR::Opcode::DsSwizzleB32: - case IR::Opcode::DsWriteAddtidB32: - case IR::Opcode::DsReadAddtidB32: return true; - default: return false; - } -} - bool ProgramNeedsFunctionLds(const IR::Program& program) { - if (program.stage == ShaderType::Compute) { + if (program.stage == ShaderType::Compute || program.values == nullptr) { return false; } - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (IsLdsOpcode(inst.op)) { - return true; + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + switch (inst.GetOpcode()) { + case IR::ValueOpcode::SwizzleU32: return true; + case IR::ValueOpcode::LoadSharedU8: + case IR::ValueOpcode::LoadSharedU16: + case IR::ValueOpcode::LoadSharedU32: + case IR::ValueOpcode::WriteSharedU8: + case IR::ValueOpcode::WriteSharedU16: + case IR::ValueOpcode::WriteSharedU32: + case IR::ValueOpcode::SharedAtomicFMin32: + case IR::ValueOpcode::SharedAtomicFMax32: + case IR::ValueOpcode::SharedAtomicSwap32: + case IR::ValueOpcode::SharedAtomicIAdd32: + case IR::ValueOpcode::SharedAtomicISub32: + case IR::ValueOpcode::SharedAtomicSMin32: + case IR::ValueOpcode::SharedAtomicUMin32: + case IR::ValueOpcode::SharedAtomicSMax32: + case IR::ValueOpcode::SharedAtomicUMax32: + case IR::ValueOpcode::SharedAtomicAnd32: + case IR::ValueOpcode::SharedAtomicOr32: + case IR::ValueOpcode::SharedAtomicXor32: + case IR::ValueOpcode::DataAppend: + case IR::ValueOpcode::DataConsume: { + const auto index = inst.Flags().index; + if (index < program.values->memory_info.size() && + program.values->memory_info[index].kind == IR::ResourceKind::Lds) { + return true; + } + break; + } + default: break; } } } @@ -332,13 +186,18 @@ bool ProgramNeedsFunctionLds(const IR::Program& program) { } bool ProgramNeedsPixelValidMask(const IR::Program& program) { - if (program.stage != ShaderType::Pixel) { + if (program.stage != ShaderType::Pixel || program.values == nullptr) { return false; } - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == IR::Opcode::Export && inst.export_info.vm) { + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() != IR::ValueOpcode::SetAttribute) { + continue; + } + const auto index = inst.Flags().index; + if (index < program.values->export_info.size() && + program.values->export_info[index].vm) { return true; } } @@ -346,20 +205,14 @@ bool ProgramNeedsPixelValidMask(const IR::Program& program) { return false; } -bool InstructionHasDppSource(const IR::Instruction& inst) { - for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) { - if (inst.src[i].dpp) { - return true; - } - } - return false; -} - bool ProgramNeedsSubgroupBallot(const IR::Program& program) { + if (program.values == nullptr) { + return false; + } if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == IR::Opcode::WqmB64) { + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() == IR::ValueOpcode::WqmMask) { return true; } } @@ -369,13 +222,22 @@ bool ProgramNeedsSubgroupBallot(const IR::Program& program) { } bool ProgramNeedsSubgroupShuffle(const IR::Program& program) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 || - inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 || - inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume || - inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) { - return true; + if (program.values == nullptr) { + return false; + } + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + switch (inst.GetOpcode()) { + case IR::ValueOpcode::DppMoveU32: + case IR::ValueOpcode::ReadFirstLane: + case IR::ValueOpcode::ReadLane: + case IR::ValueOpcode::Permlane16U32: + case IR::ValueOpcode::DataAppend: + case IR::ValueOpcode::DataConsume: + case IR::ValueOpcode::GdsDataAppend: + case IR::ValueOpcode::GdsDataConsume: + case IR::ValueOpcode::SwizzleU32: return true; + default: break; } } } @@ -383,25 +245,24 @@ bool ProgramNeedsSubgroupShuffle(const IR::Program& program) { } bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 || - inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend || - inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 || - inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export || - InstructionHasDppSource(inst)) { - return true; - } - if (IR::IsCompareOpcode(inst.op)) { - return true; - } - if (inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file == IR::RegisterFile::Vector) { - return true; - } - if (inst.dst2.kind == IR::OperandKind::Register && - inst.dst2.reg.file == IR::RegisterFile::Vector) { - return true; + if (program.values == nullptr) { + return false; + } + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + switch (inst.GetOpcode()) { + case IR::ValueOpcode::DppMoveU32: + case IR::ValueOpcode::DppUpdateU32: + case IR::ValueOpcode::WqmMask: + case IR::ValueOpcode::LaneId: + case IR::ValueOpcode::WriteLane: + case IR::ValueOpcode::Permlane16U32: + case IR::ValueOpcode::DataAppend: + case IR::ValueOpcode::DataConsume: + case IR::ValueOpcode::GdsDataAppend: + case IR::ValueOpcode::GdsDataConsume: + case IR::ValueOpcode::SwizzleU32: return true; + default: break; } } } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp new file mode 100644 index 000000000..6050fed09 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp @@ -0,0 +1,52 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { + +bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) { + if (reg.file != IR::RegisterFile::Scalar) { + return false; + } + const auto& registers = state.program.bindings.user_data_registers; + const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index); + if (found == registers.end() || *found != reg.index) { + return false; + } + dword_index = static_cast(found - registers.begin()); + return true; +} + +uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { + if (state.push_constant_variable != 0) { + const auto pointer = state.builder.AllocateId(); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer, + state.push_constant_variable, ConstantU32(state, 0), + ConstantU32(state, dword_index)}); + state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); + return value; + } + if (state.vsharp_storage_variable != 0) { + const auto pointer = state.builder.AllocateId(); + const auto value = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, + state.vsharp_storage_variable, ConstantU32(state, 0), + ConstantU32(state, dword_index)}); + state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); + return value; + } + return ConstantU32(state, 0); +} + +uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) { + if (reg.file != IR::RegisterFile::Exec) { + return 0; + } + if (state.per_invocation_masks) { + return reg.index == 0 ? 1u : 0u; + } + return reg.index == 0 || state.wave_size == 64u ? 0xffffffffu : 0u; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterExport.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterExport.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp index 94282a8af..87235fefe 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterExport.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterExport.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterFlowOps.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterFlowOps.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp index 7c6af28b0..0d89eb8a3 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterFlowOps.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlowOps.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp index 4c2b2c448..026866500 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageHelpers.cpp @@ -1,5 +1,5 @@ #include "common/assert.h" -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace { diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterImageOps.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterImageOps.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp index ae263a239..9007caed0 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterImageOps.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImageOps.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h similarity index 94% rename from src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 57bb3b8c7..db33223b5 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -4,10 +4,11 @@ #include "common/common.h" #include "common/stringUtils.h" #include "graphics/shader/recompiler/BufferFormat.h" -#include "graphics/shader/recompiler/emitter/SpirvBuilder.h" -#include "graphics/shader/recompiler/ir/BindingLayout.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include #include @@ -17,6 +18,8 @@ #include #include #include +#include +#include #include #include @@ -129,6 +132,7 @@ enum : uint32_t { OpTypeFunction = 33, OpConstant = 43, OpConstantComposite = 44, + OpUndef = 1, OpFunction = 54, OpFunctionEnd = 56, OpVariable = 59, @@ -142,6 +146,7 @@ enum : uint32_t { OpVectorShuffle = 79, OpCompositeConstruct = 80, OpCompositeExtract = 81, + OpCopyObject = 83, OpSampledImage = 86, OpImageSampleImplicitLod = 87, OpImageSampleExplicitLod = 88, @@ -153,6 +158,7 @@ enum : uint32_t { OpImageWrite = 99, OpImageQuerySizeLod = 103, OpImageQueryLod = 105, + OpImageQuerySize = 104, OpImageQueryLevels = 106, OpConvertFToU = 109, OpConvertFToS = 110, @@ -339,6 +345,11 @@ struct EmitterState { uint32_t vec3_float_type = 0; uint32_t vec4_float_type = 0; uint32_t ptr_func_uint = 0; + uint32_t ptr_func_bool = 0; + uint32_t ptr_func_float = 0; + uint32_t ptr_func_uint_pair = 0; + uint32_t ptr_func_vec4_uint = 0; + uint32_t ptr_func_vec2_float = 0; uint32_t ptr_input_float = 0; uint32_t ptr_input_bool = 0; uint32_t ptr_input_int = 0; @@ -428,6 +439,37 @@ struct EmitterState { std::map float_constants; }; +struct ValueEmitContext { + ValueEmitContext(EmitterState& state_, const IR::ValueProgram& program_) + : state(state_), program(program_) {} + + uint32_t Def(IR::Value value); + uint32_t Arg(const IR::Inst& inst, size_t index); + uint32_t Result(const IR::Inst& inst); + uint32_t TypeId(IR::Type type) const; + uint32_t Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type, + std::initializer_list args); + uint32_t Define(const IR::Inst& inst, uint32_t value); + uint32_t ResourceIndex(IR::Value value, IR::ValueOpcode opcode); + const IR::Inst* ImageAddress(IR::Value value); + const IR::MemoryInfo& Memory(const IR::Inst& inst) const; + const IR::ExportInfo& Export(const IR::Inst& inst) const; + uint32_t Label(const IR::Block* block) const; + size_t BlockIndex(const IR::Block* block) const; + void Fail(const IR::Inst& inst, const char* reason); + + EmitterState& state; + const IR::ValueProgram& program; + std::unordered_map definitions; + std::unordered_map labels; + std::unordered_map phi_variables; + std::unordered_map dispatcher_variables; + const IR::Block* current_block = nullptr; + uint32_t scratch_u32_variable = 0; + bool failed = false; + std::string error; +}; + enum class VertexInputScalarKind { Float, Sint, Uint }; constexpr uint32_t NoImageComponent = 0xffffffffu; @@ -614,22 +656,6 @@ bool IsMaskRegisterFile(IR::RegisterFile file); bool IsSccOperand(const IR::Operand& operand); -void CollectMaskStateRegisters(std::vector& registers); - -void CollectSequentialRegisters(std::vector& registers, const IR::Operand& base, - uint32_t count); - -uint32_t MaxCollectedVectorRegisterEnd(const std::vector& registers); - -void CollectMoveRelSourceRegisters(const IR::Program& program, - std::vector& registers); - -bool IsPairDwordOpcode(IR::Opcode op); - -uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count); - -void CollectRegisters(const IR::Program& program, std::vector& registers); - bool HasOutput(const std::vector& outputs, IR::StageOutputKind kind, uint32_t index); void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program); @@ -640,14 +666,10 @@ bool ProgramNeedsComputeDerivatives(const IR::Program& program); bool ProgramNeedsImageGatherExtended(const IR::Program& program); -bool IsLdsOpcode(IR::Opcode op); - bool ProgramNeedsFunctionLds(const IR::Program& program); bool ProgramNeedsPixelValidMask(const IR::Program& program); -bool InstructionHasDppSource(const IR::Instruction& inst); - bool ProgramNeedsSubgroupBallot(const IR::Program& program); bool ProgramNeedsSubgroupShuffle(const IR::Program& program); @@ -785,6 +807,11 @@ uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location); +const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location); + +uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input, + uint32_t component); + uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component); uint32_t EmitLocalInvocationIndex(EmitterState& state); @@ -887,6 +914,8 @@ uint32_t EmitSelectU32Value(EmitterState& state, uint32_t condition, uint32_t tr uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index); +bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index); + uint32_t EmitByteAddress(EmitterState& state, const IR::Instruction& inst, uint32_t first_src, uint32_t src_count); @@ -1511,49 +1540,17 @@ void EmitUpdatePixelValidMask(EmitterState& state); void EmitKillIfPixelValidMaskInactive(EmitterState& state); -void ComputeReachableBlocks(EmitterState& state, const IR::Program& program); - -void AllocateBlockLabels(EmitterState& state, const IR::Program& program); - -void AllocateDispatcherState(EmitterState& state, const IR::Program& program); - -uint32_t BlockLabel(const EmitterState& state, uint32_t block_id); - -uint32_t EmitBranchCondition(EmitterState& state, CFG::BranchCondition condition); - -void EmitStructuredPrefix(EmitterState& state, const CFG::Terminator& term); - -void EmitReturn(EmitterState& state); - -void EmitTerminator(EmitterState& state, const CFG::Terminator& term); - uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg); -void EmitRegisterVariables(EmitterState& state); +bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst); -void EmitComputeInputRegisters(EmitterState& state); +bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst); -void EmitPixelInputRegisters(EmitterState& state); +bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst); -void EmitInstruction(EmitterState& state, const IR::Instruction& inst); +bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst); -uint32_t DispatcherTargetValue(EmitterState& state, uint32_t block_id); - -void EmitDispatcherStoreTarget(EmitterState& state, uint32_t block_id); - -void EmitDispatcherExit(EmitterState& state); - -void EmitDispatcherTerminator(EmitterState& state, const CFG::Terminator& term); - -void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program); - -void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program); - -void EmitDispatcherLoopTail(EmitterState& state); - -void EmitDispatcherFunction(EmitterState& state, const IR::Program& program); - -void EmitFunction(EmitterState& state, const IR::Program& program); +bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error); // These templates accept local lambdas from several emitter translation units. template @@ -1574,7 +1571,8 @@ void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { } template -uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { +uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type, + uint32_t default_value, Fn&& fn) { if (condition == 0) { return fn(); } @@ -1595,10 +1593,16 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn& state.builder.AddFunction({OpLabel, merge_label}); const auto value = state.builder.AllocateId(); state.builder.AddFunction( - {OpPhi, state.uint_type, value, then_value, then_exit, ConstantU32(state, 0), else_label}); + {OpPhi, type, value, then_value, then_exit, default_value, else_label}); return value; } +template +uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { + return EmitValueOrDefaultIfCondition(state, condition, state.uint_type, ConstantU32(state, 0), + std::forward(fn)); +} + template void EmitGuardedByExec(EmitterState& state, Fn&& fn) { const auto then_label = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 957343d6e..0de8780d2 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { @@ -80,6 +80,21 @@ uint32_t EmitSdwaDestinationMerge(EmitterState& state, uint32_t pointer, const I state.builder.AddFunction( {OpShiftLeftLogical, state.uint_type, shifted, part, ConstantU32(state, offset)}); } + if (dst.sdwa_dst_unused == 0u) { + return shifted; + } + if (dst.sdwa_dst_unused == 1u) { + const auto extended = state.builder.AllocateId(); + const auto result = offset != 0u ? state.builder.AllocateId() : extended; + state.builder.AddFunction({OpBitFieldSExtract, state.uint_type, extended, value, + ConstantU32(state, 0), ConstantU32(state, width)}); + if (offset != 0u) { + state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, result, extended, + ConstantU32(state, offset)}); + } + return result; + } + EXIT_IF(dst.sdwa_dst_unused != 2u); state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer}); state.builder.AddFunction( {OpBitwiseAnd, state.uint_type, kept, old_value, ConstantU32(state, ~field_mask)}); @@ -367,7 +382,7 @@ uint32_t EmitMemoryByteAddress(EmitterState& state, const IR::Instruction& inst, } if (mem.kind == IR::ResourceKind::Flat || ((mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) && - state.program.info.addresses[mem.resource].source == IR::ScalarProvenance::Unknown)) { + state.program.info.addresses[mem.resource].unbased)) { return EmitFlatVirtualAddress(state, inst, first_src, src_count); } if (mem.kind == IR::ResourceKind::Global || mem.kind == IR::ResourceKind::Scratch) { diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp similarity index 97% rename from src/graphics/shader/recompiler/emitter/spirvEmitterModule.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index a149420f8..eb2170a49 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { @@ -413,6 +413,11 @@ void EmitHeaderAndTypes(EmitterState& state) { state.vec3_float_type = state.builder.AllocateId(); state.vec4_float_type = state.builder.AllocateId(); state.ptr_func_uint = state.builder.AllocateId(); + state.ptr_func_bool = state.builder.AllocateId(); + state.ptr_func_float = state.builder.AllocateId(); + state.ptr_func_uint_pair = state.builder.AllocateId(); + state.ptr_func_vec4_uint = state.builder.AllocateId(); + state.ptr_func_vec2_float = state.builder.AllocateId(); state.ptr_input_float = state.builder.AllocateId(); state.ptr_input_bool = state.builder.AllocateId(); state.ptr_input_int = state.builder.AllocateId(); @@ -568,6 +573,16 @@ void EmitHeaderAndTypes(EmitterState& state) { state.builder.AddType({OpTypeVector, state.vec4_float_type, state.float_type, 4}); state.builder.AddType( {OpTypePointer, state.ptr_func_uint, StorageClassFunction, state.uint_type}); + state.builder.AddType( + {OpTypePointer, state.ptr_func_bool, StorageClassFunction, state.bool_type}); + state.builder.AddType( + {OpTypePointer, state.ptr_func_float, StorageClassFunction, state.float_type}); + state.builder.AddType( + {OpTypePointer, state.ptr_func_uint_pair, StorageClassFunction, state.uint_pair_type}); + state.builder.AddType( + {OpTypePointer, state.ptr_func_vec4_uint, StorageClassFunction, state.vec4_uint_type}); + state.builder.AddType( + {OpTypePointer, state.ptr_func_vec2_float, StorageClassFunction, state.vec2_float_type}); state.builder.AddType( {OpTypePointer, state.ptr_input_float, StorageClassInput, state.float_type}); state.builder.AddType( diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp new file mode 100644 index 000000000..e88b8f5cc --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValue.cpp @@ -0,0 +1,541 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +uint32_t BoolConstant(EmitterState& state, bool value) { + const auto id = state.builder.AllocateId(); + state.builder.AddType({value ? 41u : 42u, state.bool_type, id}); + return id; +} + +uint32_t ConstantPair(EmitterState& state, uint64_t value) { + const auto id = state.builder.AllocateId(); + state.builder.AddType({OpConstantComposite, state.uint_pair_type, id, + ConstantU32(state, static_cast(value)), + ConstantU32(state, static_cast(value >> 32u))}); + return id; +} + +uint32_t PhiPointerType(const EmitterState& state, IR::Type type) { + switch (type) { + case IR::Type::U1: return state.ptr_func_bool; + case IR::Type::U8: + case IR::Type::U16: + case IR::Type::U32: + case IR::Type::F16: return state.ptr_func_uint; + case IR::Type::F32: return state.ptr_func_float; + case IR::Type::U64: + case IR::Type::U32x2: return state.ptr_func_uint_pair; + case IR::Type::U32x4: return state.ptr_func_vec4_uint; + case IR::Type::F32x2: return state.ptr_func_vec2_float; + default: return 0; + } +} + +void StorePhiEdge(ValueEmitContext& ctx, const IR::Block* from, const IR::Block* to) { + if (to == nullptr) { + return; + } + for (const auto& phi: *to) { + if (phi.GetOpcode() != IR::ValueOpcode::Phi) { + continue; + } + for (size_t index = 0; index < phi.NumArgs(); index++) { + if (phi.PhiBlock(index) == from) { + ctx.state.builder.AddFunction( + {OpStore, ctx.phi_variables.at(&phi), ctx.Def(phi.Arg(index))}); + break; + } + } + } +} + +const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) { + const auto found = std::ranges::find_if( + program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; }); + if (found == program.block_info.end()) { + return nullptr; + } + return program.blocks[static_cast(found - program.block_info.begin())]; +} + +void EmitReturn(ValueEmitContext& ctx) { + EmitKillIfPixelValidMaskInactive(ctx.state); + ctx.state.builder.AddFunction({OpReturn}); +} + +void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block, + const IR::ValueBlockInfo& info) { + const auto& term = info.terminator; + const auto emit_merge = [&]() { + if (term.loop_header) { + const auto* merge = TargetBlock(ctx.program, term.merge_block); + const auto* cont = TargetBlock(ctx.program, term.continue_block); + if (merge != nullptr && cont != nullptr) { + ctx.state.builder.AddFunction( + {OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone}); + } + } else if (term.kind == CFG::TerminatorKind::ConditionalBranch && + term.merge_block != UINT32_MAX) { + if (const auto* merge = TargetBlock(ctx.program, term.merge_block); merge != nullptr) { + ctx.state.builder.AddFunction( + {OpSelectionMerge, ctx.Label(merge), SelectionControlNone}); + } + } + }; + + switch (term.kind) { + case CFG::TerminatorKind::Branch: { + const auto* target = TargetBlock(ctx.program, term.true_block); + if (target == nullptr) { + EmitReturn(ctx); + return; + } + StorePhiEdge(ctx, block, target); + emit_merge(); + ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)}); + return; + } + case CFG::TerminatorKind::ConditionalBranch: { + const auto* true_block = TargetBlock(ctx.program, term.true_block); + const auto* false_block = TargetBlock(ctx.program, term.false_block); + if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) { + EmitReturn(ctx); + return; + } + StorePhiEdge(ctx, block, true_block); + StorePhiEdge(ctx, block, false_block); + const auto condition = ctx.Def(info.condition); + emit_merge(); + ctx.state.builder.AddFunction( + {OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)}); + return; + } + default: EmitReturn(ctx); return; + } +} + +void EmitDispatcherTarget(ValueEmitContext& ctx, const IR::Block* from, uint32_t target) { + const auto* block = TargetBlock(ctx.program, target); + if (block != nullptr) { + StorePhiEdge(ctx, from, block); + } +} + +void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block, + const IR::ValueBlockInfo& info, uint32_t exit_label) { + const auto& term = info.terminator; + switch (term.kind) { + case CFG::TerminatorKind::Branch: + EmitDispatcherTarget(ctx, block, term.true_block); + ctx.state.builder.AddFunction( + {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, term.true_block)}); + break; + case CFG::TerminatorKind::ConditionalBranch: { + EmitDispatcherTarget(ctx, block, term.true_block); + EmitDispatcherTarget(ctx, block, term.false_block); + const auto selected = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, selected, + ctx.Def(info.condition), + ConstantU32(ctx.state, term.true_block), + ConstantU32(ctx.state, term.false_block)}); + ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected}); + break; + } + case CFG::TerminatorKind::IndirectBranch: { + for (const auto target: term.indirect_targets) { + EmitDispatcherTarget(ctx, block, target); + } + for (const auto target: term.indirect_selector_targets) { + EmitDispatcherTarget(ctx, block, target); + } + uint32_t selected = ConstantU32(ctx.state, UINT32_MAX); + if (!info.indirect_target.IsEmpty()) { + const auto selector = ctx.Def(info.indirect_target); + const auto& values = term.indirect_selector_code != UINT32_MAX + ? term.indirect_selector_values + : term.indirect_target_pcs; + const auto& targets = term.indirect_selector_code != UINT32_MAX + ? term.indirect_selector_targets + : term.indirect_targets; + for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) { + const auto match = ctx.state.builder.AllocateId(); + const auto next = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpIEqual, ctx.state.bool_type, match, selector, + ConstantU32(ctx.state, values[index])}); + ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, next, match, + ConstantU32(ctx.state, targets[index]), + selected}); + selected = next; + } + } + ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected}); + break; + } + default: + ctx.state.builder.AddFunction( + {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)}); + break; + } + ctx.state.builder.AddFunction({OpBranch, exit_label}); +} + +bool EmitInstruction(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto finish = [&]() { + if (const auto found = ctx.dispatcher_variables.find(&inst); + found != ctx.dispatcher_variables.end()) { + ctx.state.builder.AddFunction( + {OpStore, found->second, ctx.Def(IR::Value(const_cast(&inst)))}); + } + return true; + }; + if (inst.GetOpcode() == IR::ValueOpcode::Phi) { + ctx.state.builder.AddFunction( + {OpLoad, ctx.TypeId(inst.GetType()), ctx.Result(inst), ctx.phi_variables.at(&inst)}); + return finish(); + } + if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) || + EmitValueImage(ctx, inst)) { + return finish(); + } + ctx.Fail(inst, "has no direct SPIR-V emitter"); + return false; +} + +void EmitBlock(ValueEmitContext& ctx, const IR::Block* block) { + ctx.current_block = block; + ctx.state.builder.AddFunction({OpLabel, ctx.Label(block)}); + for (const auto& inst: *block) { + if (!EmitInstruction(ctx, inst)) { + return; + } + } +} + +void EmitStructuredFunction(ValueEmitContext& ctx) { + ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())}); + for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) { + EmitBlock(ctx, ctx.program.blocks[index]); + EmitStructuredTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index]); + } +} + +void EmitDispatcherFunction(ValueEmitContext& ctx) { + auto& state = ctx.state; + const auto* entry = ctx.program.blocks.front(); + state.builder.AddFunction({OpBranch, ctx.Label(entry)}); + EmitBlock(ctx, entry); + EmitDispatcherTerminator(ctx, entry, ctx.program.block_info.front(), + state.dispatch_header_label); + + state.builder.AddFunction({OpLabel, state.dispatch_header_label}); + const auto pc = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, state.uint_type, pc, state.dispatch_pc_variable}); + const auto done = state.builder.AllocateId(); + state.builder.AddFunction( + {OpIEqual, state.bool_type, done, pc, ConstantU32(ctx.state, UINT32_MAX)}); + state.builder.AddFunction( + {OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone}); + state.builder.AddFunction( + {OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label}); + + state.builder.AddFunction({OpLabel, state.dispatch_select_label}); + state.builder.AddFunction( + {OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone}); + std::vector words {OpSwitch, pc, state.dispatch_default_label}; + for (size_t index = 1; index < ctx.program.blocks.size(); index++) { + words.push_back(ctx.program.block_info[index].id); + words.push_back(ctx.Label(ctx.program.blocks[index])); + } + state.builder.AddFunction(words); + state.builder.AddFunction({OpLabel, state.dispatch_default_label}); + state.builder.AddFunction( + {OpStore, state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)}); + state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label}); + + for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) { + EmitBlock(ctx, ctx.program.blocks[index]); + EmitDispatcherTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index], + state.dispatch_after_switch_label); + } + state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label}); + state.builder.AddFunction({OpBranch, state.dispatch_continue_label}); + state.builder.AddFunction({OpLabel, state.dispatch_continue_label}); + state.builder.AddFunction({OpBranch, state.dispatch_header_label}); + state.builder.AddFunction({OpLabel, state.dispatch_merge_label}); + EmitReturn(ctx); +} + +} // namespace + +uint32_t ValueEmitContext::TypeId(IR::Type type) const { + switch (type) { + case IR::Type::U1: return state.bool_type; + case IR::Type::U8: + case IR::Type::U16: + case IR::Type::U32: + case IR::Type::F16: return state.uint_type; + case IR::Type::U64: + case IR::Type::U32x2: return state.uint_pair_type; + case IR::Type::F32: return state.float_type; + case IR::Type::U32x4: return state.vec4_uint_type; + case IR::Type::F32x2: return state.vec2_float_type; + default: return 0; + } +} + +uint32_t ValueEmitContext::Def(IR::Value value) { + value = value.Resolve(); + if (value.IsImmediate()) { + switch (value.GetType()) { + case IR::Type::U1: return BoolConstant(state, value.U1()); + case IR::Type::U8: return ConstantU32(state, value.U8()); + case IR::Type::U16: return ConstantU32(state, value.U16()); + case IR::Type::U32: return ConstantU32(state, value.U32()); + case IR::Type::U64: return ConstantPair(state, value.U64()); + case IR::Type::F16: return ConstantU32(state, value.F16Bits()); + case IR::Type::F32: + return ConstantF32(state, std::bit_cast(value.F32Value())); + default: break; + } + } + const auto* inst = value.ResolveInstruction(); + if (inst == nullptr) { + failed = true; + error = "direct SPIR-V emitter received a non-value argument"; + return ConstantU32(state, 0); + } + if (program.dispatcher_fallback && current_block != nullptr && + inst->Parent() != current_block) { + if (const auto found = dispatcher_variables.find(inst); + found != dispatcher_variables.end()) { + const auto id = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second}); + return id; + } + } + if (const auto found = definitions.find(inst); found != definitions.end()) { + return found->second; + } + const auto id = state.builder.AllocateId(); + definitions.emplace(inst, id); + return id; +} + +uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) { + return Def(inst.Arg(index)); +} + +uint32_t ValueEmitContext::Result(const IR::Inst& inst) { + if (const auto found = definitions.find(&inst); found != definitions.end()) { + return found->second; + } + const auto id = state.builder.AllocateId(); + definitions.emplace(&inst, id); + return id; +} + +uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type, + std::initializer_list args) { + std::vector words {opcode, TypeId(type), Result(inst)}; + words.insert(words.end(), args.begin(), args.end()); + state.builder.AddFunction(words); + return Result(inst); +} + +uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) { + if (const auto found = definitions.find(&inst); found != definitions.end()) { + if (found->second != value) { + state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value}); + } + return found->second; + } + definitions.emplace(&inst, value); + return value; +} + +uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) { + const auto* inst = value.ResolveInstruction(); + if (inst == nullptr || inst->GetOpcode() != opcode) { + failed = true; + error = "typed resource handle has the wrong producer"; + return 0; + } + return inst->Flags(); +} + +const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) { + const auto* inst = value.ResolveInstruction(); + if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) { + failed = true; + error = "typed image address was not constructed by MakeImageAddress"; + return nullptr; + } + return inst; +} + +const IR::MemoryInfo& ValueEmitContext::Memory(const IR::Inst& inst) const { + return program.memory_info.at(inst.Flags().index); +} + +const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const { + return program.export_info.at(inst.Flags().index); +} + +uint32_t ValueEmitContext::Label(const IR::Block* block) const { + return labels.at(block); +} + +size_t ValueEmitContext::BlockIndex(const IR::Block* block) const { + return static_cast(std::ranges::find(program.blocks, block) - program.blocks.begin()); +} + +void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) { + failed = true; + error = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason); +} + +bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) { + ValueEmitContext ctx(state, program); + for (const auto* block: program.blocks) { + ctx.labels.emplace(block, state.builder.AllocateId()); + for (const auto& inst: *block) { + if (inst.GetOpcode() != IR::ValueOpcode::Phi) { + continue; + } + const auto ptr_type = PhiPointerType(state, inst.GetType()); + if (ptr_type == 0) { + ctx.failed = true; + ctx.error = fmt::format("typed opcode Phi of type {} cannot be spilled across a " + "control-flow edge", + IR::TypeName(inst.GetType())); + break; + } + ctx.phi_variables.emplace(&inst, state.builder.AllocateId()); + } + } + if (ctx.failed) { + SetError(error, ctx.error.c_str()); + return false; + } + if (program.dispatcher_fallback) { + const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) { + value = value.Resolve(); + const auto* definition = value.TryInstruction(); + if (definition == nullptr || definition->Parent() == consumer || + PhiPointerType(state, definition->GetType()) == 0) { + return; + } + if (!ctx.dispatcher_variables.contains(definition)) { + ctx.dispatcher_variables.emplace(definition, state.builder.AllocateId()); + } + }; + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + for (size_t index = 0; index < inst.NumArgs(); index++) { + const auto* consumer = + inst.GetOpcode() == IR::ValueOpcode::Phi ? inst.PhiBlock(index) : block; + mark_cross_block(inst.Arg(index), consumer); + } + } + } + for (size_t index = 0; index < program.blocks.size(); index++) { + mark_cross_block(program.block_info[index].condition, program.blocks[index]); + mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]); + } + state.dispatcher_fallback = true; + state.dispatch_pc_variable = state.builder.AllocateId(); + state.dispatch_header_label = state.builder.AllocateId(); + state.dispatch_select_label = state.builder.AllocateId(); + state.dispatch_default_label = state.builder.AllocateId(); + state.dispatch_after_switch_label = state.builder.AllocateId(); + state.dispatch_continue_label = state.builder.AllocateId(); + state.dispatch_merge_label = state.builder.AllocateId(); + } + for (const auto* block: program.blocks) { + if (std::ranges::any_of(*block, [](const IR::Inst& inst) { + return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 || + inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 || + inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 || + inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 || + inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32; + })) { + ctx.scratch_u32_variable = state.builder.AllocateId(); + break; + } + } + state.builder.AddFunction( + {OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type}); + state.builder.AddFunction({OpLabel, state.entry_label}); + if (state.needs_function_lds) { + state.builder.AddFunction( + {OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction}); + } + if (state.pixel_valid_mask_variable != 0) { + state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable, + StorageClassFunction}); + } + if (program.dispatcher_fallback) { + state.builder.AddFunction( + {OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction}); + } + for (const auto& binding: state.registers) { + state.builder.AddFunction( + {OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction}); + } + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() != IR::ValueOpcode::Phi) { + continue; + } + state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()), + ctx.phi_variables.at(&inst), StorageClassFunction}); + } + } + if (program.dispatcher_fallback) { + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + if (const auto found = ctx.dispatcher_variables.find(&inst); + found != ctx.dispatcher_variables.end()) { + state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()), + found->second, StorageClassFunction}); + } + } + } + } + if (ctx.scratch_u32_variable != 0) { + state.builder.AddFunction( + {OpVariable, state.ptr_func_uint, ctx.scratch_u32_variable, StorageClassFunction}); + } + for (const auto& binding: state.registers) { + state.builder.AddFunction({OpStore, binding.pointer_id, + ConstantU32(state, InitialRegisterValue(state, binding.reg))}); + } + if (state.pixel_valid_mask_variable != 0) { + state.builder.AddFunction( + {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)}); + } + EmitStorageBufferOffsets(state); + if (program.blocks.empty()) { + EmitReturn(ctx); + } else if (program.dispatcher_fallback) { + EmitDispatcherFunction(ctx); + } else { + EmitStructuredFunction(ctx); + } + state.builder.AddFunction({OpFunctionEnd}); + if (ctx.failed) { + SetError(error, ctx.error.c_str()); + return false; + } + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAlu.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAlu.cpp new file mode 100644 index 000000000..e991c2af8 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueAlu.cpp @@ -0,0 +1,677 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +struct Pair { + uint32_t low = 0; + uint32_t high = 0; +}; + +uint32_t NewUnary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, type, result, value}); + return result; +} + +uint32_t NewBinary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, + uint32_t rhs) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, type, result, lhs, rhs}); + return result; +} + +uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value, + uint32_t false_value) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value}); + return result; +} + +Pair ExtractPair(EmitterState& state, uint32_t value) { + Pair result {state.builder.AllocateId(), state.builder.AllocateId()}; + state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.low, value, 0}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.high, value, 1}); + return result; +} + +uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeConstruct, state.uint_pair_type, result, low, high}); + return result; +} + +uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, + uint32_t high_compare, uint32_t low_compare, bool equal_compare = false, + bool not_equal = false) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto high_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.high, rhs.high); + const auto low_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.low, rhs.low); + if (equal_compare) { + return NewBinary( + state, not_equal ? OpLogicalOr : OpLogicalAnd, state.bool_type, + not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, high_equal) : high_equal, + not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, low_equal) : low_equal); + } + const auto high_result = NewBinary(state, high_compare, state.bool_type, lhs.high, rhs.high); + const auto low_result = NewBinary(state, low_compare, state.bool_type, lhs.low, rhs.low); + const auto low_path = NewBinary(state, OpLogicalAnd, state.bool_type, high_equal, low_result); + return NewBinary(state, OpLogicalOr, state.bool_type, high_result, low_path); +} + +uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0)); + const auto high0 = NewBinary(state, OpIAdd, state.uint_type, lhs.high, rhs.high); + const auto high = NewBinary(state, OpIAdd, state.uint_type, high0, carry.carry); + return MakePair(state, carry.sum, high); +} + +uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto low = NewBinary(state, OpISub, state.uint_type, lhs.low, rhs.low); + const auto borrow = NewBinary(state, OpULessThan, state.bool_type, lhs.low, rhs.low); + const auto borrow_u32 = + NewSelect(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0)); + const auto high0 = NewBinary(state, OpISub, state.uint_type, lhs.high, rhs.high); + const auto high = NewBinary(state, OpISub, state.uint_type, high0, borrow_u32); + return MakePair(state, low, high); +} + +uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) { + const auto operand_type = signed_value ? state.int_type : state.uint_type; + const auto pair_type = signed_value ? state.int_pair_type : state.uint_pair_type; + uint32_t lhs_operand = lhs; + uint32_t rhs_operand = rhs; + if (signed_value) { + lhs_operand = NewUnary(state, OpBitcast, state.int_type, lhs); + rhs_operand = NewUnary(state, OpBitcast, state.int_type, rhs); + } + const auto extended = state.builder.AllocateId(); + state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended, + lhs_operand, rhs_operand}); + const auto high = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1}); + return signed_value ? NewUnary(state, OpBitcast, state.uint_type, high) : high; +} + +uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + const auto low = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.low); + const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false); + const auto high1 = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.high); + const auto high2 = NewBinary(state, OpIMul, state.uint_type, lhs.high, rhs.low); + return MakePair(state, low, + NewBinary(state, OpIAdd, state.uint_type, + NewBinary(state, OpIAdd, state.uint_type, high0, high1), high2)); +} + +uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value, + uint32_t rhs_value) { + const auto lhs = ExtractPair(state, lhs_value); + const auto rhs = ExtractPair(state, rhs_value); + return MakePair(state, NewBinary(state, opcode, state.uint_type, lhs.low, rhs.low), + NewBinary(state, opcode, state.uint_type, lhs.high, rhs.high)); +} + +uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { + const auto pair = ExtractPair(state, value); + uint32_t low = 0; + uint32_t high = 0; + if (opcode == OpShiftLeftLogical) { + EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high); + } else if (opcode == OpShiftRightLogical) { + EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high); + } else { + const auto logical = EmitShift64(state, OpShiftRightLogical, value, shift); + const auto logical_pair = ExtractPair(state, logical); + const auto sign = + NewBinary(state, OpSLessThan, state.bool_type, pair.high, ConstantU32(state, 0)); + const auto inverse = + NewBinary(state, OpISub, state.uint_type, ConstantU32(state, 64), shift); + const auto fill = NewBinary(state, OpShiftLeftLogical, state.uint_type, + ConstantU32(state, 0xffffffffu), inverse); + const auto high_shift = + NewBinary(state, OpISub, state.uint_type, shift, ConstantU32(state, 32)); + const auto high_fill = NewSelect( + state, state.uint_type, + NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, ConstantU32(state, 32)), + ConstantU32(state, 0xffffffffu), fill); + const auto fill_low = + NewSelect(state, state.uint_type, sign, + NewSelect(state, state.uint_type, + NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, + ConstantU32(state, 32)), + NewBinary(state, OpShiftLeftLogical, state.uint_type, + ConstantU32(state, 0xffffffffu), high_shift), + ConstantU32(state, 0)), + ConstantU32(state, 0)); + const auto fill_high = + NewSelect(state, state.uint_type, sign, high_fill, ConstantU32(state, 0)); + return MakePair( + state, NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.low, fill_low), + NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.high, fill_high)); + } + return MakePair(state, low, high); +} + +uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) { + const auto pair = ExtractPair(state, value); + const auto high_i = state.builder.AllocateId(); + const auto low_i = state.builder.AllocateId(); + state.builder.AddFunction( + {OpExtInst, state.int_type, high_i, state.glsl_std450, GlslFindUMsb, pair.high}); + state.builder.AddFunction( + {OpExtInst, state.int_type, low_i, state.glsl_std450, GlslFindUMsb, pair.low}); + const auto high = NewUnary(state, OpBitcast, state.uint_type, high_i); + const auto low = NewUnary(state, OpBitcast, state.uint_type, low_i); + const auto high_nonzero = + NewBinary(state, OpINotEqual, state.bool_type, pair.high, ConstantU32(state, 0)); + return NewSelect(state, state.uint_type, high_nonzero, + NewBinary(state, OpIAdd, state.uint_type, high, ConstantU32(state, 32)), low); +} + +uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value, + bool max_value) { + const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value) + : EmitMinMaxU32Value(state, a, b, max_value); + return signed_value ? EmitMinMaxI32Value(state, ab, c, max_value) + : EmitMinMaxU32Value(state, ab, c, max_value); +} + +uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) { + const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false); + const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true); + const auto ab = NewBinary(state, OpIAdd, state.uint_type, a, b); + const auto abc = NewBinary(state, OpIAdd, state.uint_type, ab, c); + return NewBinary(state, OpISub, state.uint_type, + NewBinary(state, OpISub, state.uint_type, abc, minimum), maximum); +} + +uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) { + return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value); +} + +uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) { + const auto min_ab = EmitMinMaxF32Value(state, a, b, false); + const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false); + const auto max_ab = EmitMinMaxF32Value(state, a, b, true); + const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false); + const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true); + const auto nan_ab = NewBinary(state, OpLogicalOr, state.bool_type, + EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan); + const auto any_nan = + NewBinary(state, OpLogicalOr, state.bool_type, nan_ab, EmitClassifyF32(state, c).nan); + return NewSelect(state, state.float_type, any_nan, min3, median); +} + +uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode, + std::initializer_list args) { + const auto result = state.builder.AllocateId(); + std::vector words {OpExtInst, type, result, state.glsl_std450, opcode}; + words.insert(words.end(), args.begin(), args.end()); + state.builder.AddFunction(words); + return result; +} + +uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) { + const auto trunc = EmitTruncF32Value(state, src); + const auto converted_raw = state.builder.AllocateId(); + if (signed_value) { + const auto converted_i = state.builder.AllocateId(); + state.builder.AddFunction({OpConvertFToS, state.int_type, converted_i, trunc}); + state.builder.AddFunction({OpBitcast, state.uint_type, converted_raw, converted_i}); + } else { + state.builder.AddFunction({OpConvertFToU, state.uint_type, converted_raw, trunc}); + } + const auto nan = EmitClassifyF32(state, src).nan; + if (signed_value) { + const auto below = NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, + ConstantF32(state, 0xcf000000u)); + const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src, + ConstantF32(state, 0x4f000000u)); + const auto high = NewSelect(state, state.uint_type, above, ConstantU32(state, 0x7fffffffu), + converted_raw); + const auto low = + NewSelect(state, state.uint_type, below, ConstantU32(state, 0x80000000u), high); + return NewSelect(state, state.uint_type, nan, ConstantU32(state, 0), low); + } + const auto below = + NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, ConstantF32(state, 0)); + const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src, + ConstantF32(state, 0x4f800000u)); + const auto zero = NewBinary(state, OpLogicalOr, state.bool_type, nan, below); + const auto high = + NewSelect(state, state.uint_type, above, ConstantU32(state, 0xffffffffu), converted_raw); + return NewSelect(state, state.uint_type, zero, ConstantU32(state, 0), high); +} + +uint32_t EmitPackHalf(EmitterState& state, uint32_t src) { + return EmitExt(state, state.uint_type, GlslPackHalf2x16, {src}); +} + +uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) { + const auto pair = EmitExt(state, state.vec2_float_type, GlslUnpackHalf2x16, {bits}); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, state.float_type, result, pair, 0}); + return result; +} + +} // namespace + +bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto op = inst.GetOpcode(); + const auto unary = [&](uint32_t spirv, IR::Type type) { + ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0)}); + return true; + }; + const auto binary = [&](uint32_t spirv, IR::Type type) { + ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; + }; + const auto ext_unary = [&](uint32_t ext) { + ctx.Define(inst, EmitExt(state, state.float_type, ext, {ctx.Arg(inst, 0)})); + return true; + }; + switch (op) { + case IR::ValueOpcode::BitCastU16F16: + case IR::ValueOpcode::BitCastF16U16: + case IR::ValueOpcode::ConvertU32U16: + case IR::ValueOpcode::ConvertU32U8: + case IR::ValueOpcode::PackUint2x32: + case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true; + case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32); + case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32); + case IR::ValueOpcode::ConvertU16U32: + ctx.Emit(inst, OpBitwiseAnd, IR::Type::U16, + {ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)}); + return true; + case IR::ValueOpcode::ConvertU8U32: + ctx.Emit(inst, OpBitwiseAnd, IR::Type::U8, + {ctx.Arg(inst, 0), ConstantU32(state, 0xffu)}); + return true; + case IR::ValueOpcode::ConvertF16F32: { + const auto pair = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair, + ctx.Arg(inst, 0), ConstantF32(state, 0)}); + ctx.Define(inst, EmitPackHalf(state, pair)); + return true; + } + case IR::ValueOpcode::ConvertF32F16: + ctx.Define(inst, EmitUnpackHalf(state, ctx.Arg(inst, 0))); + return true; + case IR::ValueOpcode::ConvertS32F32: + ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), true)); + return true; + case IR::ValueOpcode::ConvertU32F32: + ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false)); + return true; + case IR::ValueOpcode::ConvertF32S32: { + const auto signed_value = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0)); + ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value}); + return true; + } + case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32); + case IR::ValueOpcode::CompositeConstructU32x2: + ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; + case IR::ValueOpcode::CompositeConstructF32x2: + ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; + case IR::ValueOpcode::CompositeConstructU32x4: + ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}); + return true; + case IR::ValueOpcode::CompositeExtractU32x2: + case IR::ValueOpcode::CompositeExtractU32x4: + ctx.Emit(inst, OpCompositeExtract, IR::Type::U32, + {ctx.Arg(inst, 0), inst.Arg(1).U32()}); + return true; + case IR::ValueOpcode::PackHalf2x16: + ctx.Define(inst, EmitExt(state, state.uint_type, GlslPackHalf2x16, {ctx.Arg(inst, 0)})); + return true; + case IR::ValueOpcode::PackSnorm2x16: + ctx.Define(inst, + EmitExt(state, state.uint_type, GlslPackSnorm2x16, {ctx.Arg(inst, 0)})); + return true; + case IR::ValueOpcode::PackUnorm2x16: + ctx.Define(inst, + EmitExt(state, state.uint_type, GlslPackUnorm2x16, {ctx.Arg(inst, 0)})); + return true; + case IR::ValueOpcode::PackFloat2x16Rtz: { + const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0)); + const auto high = + NewBinary(state, OpShiftLeftLogical, state.uint_type, + EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16)); + ctx.Define(inst, NewBinary(state, OpBitwiseOr, state.uint_type, low, high)); + return true; + } + case IR::ValueOpcode::FPAbs32: + ctx.Define(inst, EmitFAbsValue(state, ctx.Arg(inst, 0))); + return true; + case IR::ValueOpcode::FPNeg32: + ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0))); + return true; + case IR::ValueOpcode::FPSaturate32: + ctx.Define(inst, EmitExt(state, state.float_type, GlslFClamp, + {ctx.Arg(inst, 0), ConstantF32(state, 0), + ConstantF32(state, 0x3f800000u)})); + return true; + case IR::ValueOpcode::BitFieldInsert: + ctx.Emit(inst, OpBitFieldInsert, IR::Type::U32, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}); + return true; + case IR::ValueOpcode::BitFieldUExtract: + ctx.Emit(inst, OpBitFieldUExtract, IR::Type::U32, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); + return true; + case IR::ValueOpcode::BitFieldSExtract: + ctx.Emit(inst, OpBitFieldSExtract, IR::Type::U32, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); + return true; + case IR::ValueOpcode::SelectU1: + ctx.Emit(inst, OpSelect, IR::Type::U1, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); + return true; + case IR::ValueOpcode::SelectU32: + ctx.Emit(inst, OpSelect, IR::Type::U32, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); + return true; + case IR::ValueOpcode::SelectF32: + ctx.Emit(inst, OpSelect, IR::Type::F32, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); + return true; + case IR::ValueOpcode::IAdd32: return binary(OpIAdd, IR::Type::U32); + case IR::ValueOpcode::ISub32: return binary(OpISub, IR::Type::U32); + case IR::ValueOpcode::IMul32: return binary(OpIMul, IR::Type::U32); + case IR::ValueOpcode::UDiv32: return binary(OpUDiv, IR::Type::U32); + case IR::ValueOpcode::IAdd64: + ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::ISub64: + ctx.Define(inst, EmitSub64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::IMul64: + ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::IAddCarry32: + ctx.Emit(inst, OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; + case IR::ValueOpcode::SMulHi: + ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true)); + return true; + case IR::ValueOpcode::UMulHi: + ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false)); + return true; + case IR::ValueOpcode::IAbs32: { + const auto value = ctx.Arg(inst, 0); + const auto neg = NewUnary(state, OpSNegate, state.uint_type, value); + const auto negative = + NewBinary(state, OpSLessThan, state.bool_type, value, ConstantU32(state, 0)); + ctx.Define(inst, NewSelect(state, state.uint_type, negative, neg, value)); + return true; + } + case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32); + case IR::ValueOpcode::ShiftRightLogical32: + return binary(OpShiftRightLogical, IR::Type::U32); + case IR::ValueOpcode::ShiftRightArithmetic32: + return binary(OpShiftRightArithmetic, IR::Type::U32); + case IR::ValueOpcode::ShiftLeftLogical64: + ctx.Define(inst, + EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::ShiftRightLogical64: + ctx.Define(inst, + EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::ShiftRightArithmetic64: + ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0), + ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32); + case IR::ValueOpcode::BitwiseOr32: return binary(OpBitwiseOr, IR::Type::U32); + case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32); + case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32); + case IR::ValueOpcode::BitwiseAnd64: + ctx.Define(inst, + EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::BitwiseOr64: + ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32); + case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32); + case IR::ValueOpcode::BitCount64: { + const auto pair = ExtractPair(state, ctx.Arg(inst, 0)); + ctx.Define(inst, NewBinary(state, OpIAdd, state.uint_type, + NewUnary(state, OpBitCount, state.uint_type, pair.low), + NewUnary(state, OpBitCount, state.uint_type, pair.high))); + return true; + } + case IR::ValueOpcode::FindILsb32: { + const auto value = EmitExt(state, state.int_type, GlslFindILsb, {ctx.Arg(inst, 0)}); + ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value)); + return true; + } + case IR::ValueOpcode::FindUMsb32: { + const auto value = EmitExt(state, state.int_type, GlslFindUMsb, {ctx.Arg(inst, 0)}); + ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value)); + return true; + } + case IR::ValueOpcode::FindUMsb64: + ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0))); + return true; + case IR::ValueOpcode::FindSMsb32: { + const auto value = + EmitExt(state, state.int_type, 74u, + {NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0))}); + ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value)); + return true; + } + case IR::ValueOpcode::SMin32: + ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false)); + return true; + case IR::ValueOpcode::SMax32: + ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true)); + return true; + case IR::ValueOpcode::UMin32: + ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false)); + return true; + case IR::ValueOpcode::UMax32: + ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true)); + return true; + case IR::ValueOpcode::SMinTri32: + ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + ctx.Arg(inst, 2), true, false)); + return true; + case IR::ValueOpcode::SMaxTri32: + ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + ctx.Arg(inst, 2), true, true)); + return true; + case IR::ValueOpcode::UMinTri32: + ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + ctx.Arg(inst, 2), false, false)); + return true; + case IR::ValueOpcode::UMaxTri32: + ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + ctx.Arg(inst, 2), false, true)); + return true; + case IR::ValueOpcode::SMedTri32: + ctx.Define(inst, + EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), true)); + return true; + case IR::ValueOpcode::UMedTri32: + ctx.Define( + inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false)); + return true; + case IR::ValueOpcode::SLessThan32: return binary(OpSLessThan, IR::Type::U1); + case IR::ValueOpcode::ULessThan32: return binary(OpULessThan, IR::Type::U1); + case IR::ValueOpcode::IEqual32: return binary(OpIEqual, IR::Type::U1); + case IR::ValueOpcode::SLessThanEqual32: return binary(OpSLessThanEqual, IR::Type::U1); + case IR::ValueOpcode::ULessThanEqual32: return binary(OpULessThanEqual, IR::Type::U1); + case IR::ValueOpcode::SGreaterThan32: return binary(OpSGreaterThan, IR::Type::U1); + case IR::ValueOpcode::UGreaterThan32: return binary(OpUGreaterThan, IR::Type::U1); + case IR::ValueOpcode::INotEqual32: return binary(OpINotEqual, IR::Type::U1); + case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1); + case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1); + case IR::ValueOpcode::IEqual64: + ctx.Define(inst, + Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false)); + return true; + case IR::ValueOpcode::INotEqual64: + ctx.Define(inst, + Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true)); + return true; + case IR::ValueOpcode::ULessThan64: + ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, + OpULessThan)); + return true; + case IR::ValueOpcode::SLessThan64: + ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, + OpULessThan)); + return true; + case IR::ValueOpcode::UGreaterThan64: + ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan, + OpUGreaterThan)); + return true; + case IR::ValueOpcode::SGreaterThan64: + ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan, + OpUGreaterThan)); + return true; + case IR::ValueOpcode::ULessThanEqual64: { + const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpUGreaterThan, OpUGreaterThan); + ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater)); + return true; + } + case IR::ValueOpcode::SLessThanEqual64: { + const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + OpSGreaterThan, OpUGreaterThan); + ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater)); + return true; + } + case IR::ValueOpcode::UGreaterThanEqual64: { + const auto less = + Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan); + ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less)); + return true; + } + case IR::ValueOpcode::SGreaterThanEqual64: { + const auto less = + Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan); + ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less)); + return true; + } + case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1); + case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1); + case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1); + case IR::ValueOpcode::LogicalNot: return unary(OpLogicalNot, IR::Type::U1); + case IR::ValueOpcode::FPOrdEqual32: return binary(OpFOrdEqual, IR::Type::U1); + case IR::ValueOpcode::FPUnordEqual32: return binary(OpFUnordEqual, IR::Type::U1); + case IR::ValueOpcode::FPOrdNotEqual32: return binary(OpFOrdNotEqual, IR::Type::U1); + case IR::ValueOpcode::FPUnordNotEqual32: return binary(OpFUnordNotEqual, IR::Type::U1); + case IR::ValueOpcode::FPOrdLessThan32: return binary(OpFOrdLessThan, IR::Type::U1); + case IR::ValueOpcode::FPUnordLessThan32: return binary(OpFUnordLessThan, IR::Type::U1); + case IR::ValueOpcode::FPOrdGreaterThan32: return binary(OpFOrdGreaterThan, IR::Type::U1); + case IR::ValueOpcode::FPUnordGreaterThan32: + return binary(OpFUnordGreaterThan, IR::Type::U1); + case IR::ValueOpcode::FPOrdLessThanEqual32: + return binary(OpFOrdLessThanEqual, IR::Type::U1); + case IR::ValueOpcode::FPUnordLessThanEqual32: + return binary(OpFUnordLessThanEqual, IR::Type::U1); + case IR::ValueOpcode::FPOrdGreaterThanEqual32: + return binary(OpFOrdGreaterThanEqual, IR::Type::U1); + case IR::ValueOpcode::FPUnordGreaterThanEqual32: + return binary(OpFUnordGreaterThanEqual, IR::Type::U1); + case IR::ValueOpcode::FPIsNan32: + ctx.Emit(inst, OpFUnordNotEqual, IR::Type::U1, {ctx.Arg(inst, 0), ctx.Arg(inst, 0)}); + return true; + case IR::ValueOpcode::FPCmpClass32: + ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::FPAdd32: return binary(OpFAdd, IR::Type::F32); + case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32); + case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32); + case IR::ValueOpcode::FPFma32: + ctx.Define(inst, EmitExt(state, state.float_type, GlslFma, + {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)})); + return true; + case IR::ValueOpcode::FPMin32: + ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false)); + return true; + case IR::ValueOpcode::FPMax32: + ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true)); + return true; + case IR::ValueOpcode::FPMinTri32: + ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + ctx.Arg(inst, 2), false)); + return true; + case IR::ValueOpcode::FPMaxTri32: + ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), + ctx.Arg(inst, 2), true)); + return true; + case IR::ValueOpcode::FPMedTri32: + ctx.Define(inst, + EmitFMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2))); + return true; + case IR::ValueOpcode::FPRecip32: { + const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0)); + ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type, + ConstantF32(state, 0x3f800000u), source)); + return true; + } + case IR::ValueOpcode::FPRecipIFlag32: + // Integer-to-float inputs used by IFLAG cannot be denormal. + ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type, + ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0))); + return true; + case IR::ValueOpcode::FPRecipSqrt32: + ctx.Define(inst, EmitExt(state, state.float_type, GlslInverseSqrt, + {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); + return true; + case IR::ValueOpcode::FPSqrt: + ctx.Define(inst, EmitExt(state, state.float_type, GlslSqrt, + {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); + return true; + case IR::ValueOpcode::FPSin: + case IR::ValueOpcode::FPCos: { + auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin); + source = + NewBinary(state, OpFMul, state.float_type, source, ConstantF32(state, 0x40c90fdbu)); + ctx.Define(inst, EmitExt(state, state.float_type, + op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source})); + return true; + } + case IR::ValueOpcode::FPExp2: + ctx.Define(inst, EmitExt(state, state.float_type, GlslExp2, + {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); + return true; + case IR::ValueOpcode::FPLog2: + ctx.Define(inst, EmitExt(state, state.float_type, GlslLog2, + {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); + return true; + case IR::ValueOpcode::FPLdexp: { + const auto exponent = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 1)); + ctx.Define(inst, + EmitExt(state, state.float_type, GlslLdexp, {ctx.Arg(inst, 0), exponent})); + return true; + } + case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven); + case IR::ValueOpcode::FPFloor32: return ext_unary(GlslFloor); + case IR::ValueOpcode::FPCeil32: return ext_unary(GlslCeil); + case IR::ValueOpcode::FPTrunc32: return ext_unary(GlslTrunc); + case IR::ValueOpcode::FPFract32: return ext_unary(GlslFract); + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp new file mode 100644 index 000000000..cb383fa72 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueFlow.cpp @@ -0,0 +1,557 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) { + const auto result = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value, + ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)}); + return result; +} + +void StoreRegisterMirror(ValueEmitContext& ctx, IR::Register reg, uint32_t value) { + if (const auto pointer = PointerForRegister(ctx.state, reg); pointer != 0) { + if (IsInactiveWave32ExecHigh(ctx.state, reg)) { + value = ConstantU32(ctx.state, 0); + } + ctx.state.builder.AddFunction({OpStore, pointer, value}); + } +} + +bool IsPrologue(const ValueEmitContext& ctx) { + return !ctx.program.blocks.empty() && ctx.current_block == ctx.program.blocks.front(); +} + +uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) { + auto& state = ctx.state; + if (kind == IR::StageInputKind::LocalInvocationIndex) { + return EmitLocalInvocationIndex(state); + } + const auto variable = InputVariableForKind(state, kind); + if (variable == 0) { + return ConstantU32(state, 0); + } + if (kind == IR::StageInputKind::FrontFacing) { + const auto value = state.builder.AllocateId(); + const auto bits = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, state.bool_type, value, variable}); + state.builder.AddFunction( + {OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)}); + return bits; + } + if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) { + const auto value = state.builder.AllocateId(); + const auto bits = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, state.int_type, value, variable}); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, value}); + return bits; + } + if (kind == IR::StageInputKind::FragCoord) { + const auto pointer = state.builder.AllocateId(); + const auto value = state.builder.AllocateId(); + const auto bits = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_input_float, pointer, variable, + ConstantU32(state, component)}); + state.builder.AddFunction({OpLoad, state.float_type, value, pointer}); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, value}); + return bits; + } + return EmitInputComponentU32(state, kind, component); +} + +uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) { + auto& state = ctx.state; + const auto ballot = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), active}); + const auto low = state.builder.AllocateId(); + const auto high = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1}); + const auto wqm_low = EmitWqmLaneU32(state, low); + const auto wqm_high = EmitWqmLaneU32(state, high); + const auto lane = EmitSubgroupLocalInvocationId(state); + const auto upper = state.builder.AllocateId(); + const auto mask = state.builder.AllocateId(); + const auto bit_lane = state.builder.AllocateId(); + const auto bit = state.builder.AllocateId(); + const auto hit = state.builder.AllocateId(); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction( + {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 32)}); + state.builder.AddFunction({OpSelect, state.uint_type, mask, upper, wqm_high, wqm_low}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, bit_lane, lane, ConstantU32(state, 31)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_lane}); + state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit}); + state.builder.AddFunction({OpINotEqual, state.bool_type, result, hit, ConstantU32(state, 0)}); + return result; +} + +uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags, + uint32_t exec) { + auto& state = ctx.state; + const auto lane = EmitSubgroupLocalInvocationId(state); + const auto bank_shift = state.builder.AllocateId(); + const auto row_shift = state.builder.AllocateId(); + const auto bank = state.builder.AllocateId(); + const auto row = state.builder.AllocateId(); + const auto bank_bit = state.builder.AllocateId(); + const auto row_bit = state.builder.AllocateId(); + const auto bank_hit = state.builder.AllocateId(); + const auto row_hit = state.builder.AllocateId(); + const auto bank_ok = state.builder.AllocateId(); + const auto row_ok = state.builder.AllocateId(); + const auto masks_ok = state.builder.AllocateId(); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, bank_shift, lane, ConstantU32(state, 2)}); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, row_shift, lane, ConstantU32(state, 4)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, flags.bank_mask), bank_bit}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, flags.row_mask), row_bit}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, bank_ok, bank_hit, ConstantU32(state, 0)}); + state.builder.AddFunction( + {OpINotEqual, state.bool_type, row_ok, row_hit, ConstantU32(state, 0)}); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, masks_ok, bank_ok, row_ok}); + uint32_t writable = masks_ok; + if (!flags.bound_control) { + const auto target = EmitDppTargetLane(state, flags.control); + const auto bounded = state.builder.AllocateId(); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, bounded, writable, target.valid}); + writable = bounded; + } + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpLogicalAnd, state.bool_type, result, exec, writable}); + return result; +} + +uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) { + auto& state = ctx.state; + const auto* input = InputBindingForParameter(state, attr); + if (input == nullptr || input->variable_id == 0) { + return ConstantU32(state, 0); + } + if (state.stage == ShaderType::Vertex) { + return EmitVertexParameterComponentU32(state, *input, chan & 3u); + } + const auto vector = state.builder.AllocateId(); + const auto component = state.builder.AllocateId(); + const auto bits = state.builder.AllocateId(); + state.builder.AddFunction({OpLoad, state.vec4_float_type, vector, input->variable_id}); + state.builder.AddFunction({OpCompositeExtract, state.float_type, component, vector, chan & 3u}); + state.builder.AddFunction({OpBitcast, state.uint_type, bits, component}); + return bits; +} + +bool MrtUsesUint(const EmitterState& state, const IR::ExportInfo& exp) { + return exp.kind == IR::ExportTargetKind::Mrt && state.pixel_input_info != nullptr && + exp.index < std::size(state.pixel_input_info->target_output_mode) && + state.pixel_input_info->target_output_mode[exp.index] == 7u; +} + +uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) { + const auto value = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpCompositeExtract, ctx.state.uint_type, value, vector, component}); + return value; +} + +uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp, + bool uint_output) { + auto& state = ctx.state; + if (exp.compr && !uint_output) { + uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0), + ConstantF32(state, 0x3f800000u)}; + for (uint32_t pair = 0; pair < 2u; pair++) { + if ((exp.en & (3u << (pair * 2u))) == 0u) { + continue; + } + const auto packed = state.builder.AllocateId(); + const auto unpacked = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, packed, data, pair}); + state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, + state.glsl_std450, GlslUnpackHalf2x16, packed}); + for (uint32_t lane = 0; lane < 2u; lane++) { + const auto component = pair * 2u + lane; + if (((exp.en >> component) & 1u) != 0u) { + f32[component] = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeExtract, state.float_type, f32[component], unpacked, lane}); + } + } + } + const auto vector = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]}); + return vector; + } + uint32_t raw[4] = { + ConstantU32(state, 0), + ConstantU32(state, 0), + ConstantU32(state, 0), + ConstantU32(state, uint_output ? 1u : 0x3f800000u), + }; + if (exp.compr) { + for (uint32_t pair = 0; pair < 2u; pair++) { + if ((exp.en & (3u << (pair * 2u))) == 0u) { + continue; + } + const auto packed = ExportRawComponent(ctx, data, pair); + for (uint32_t lane = 0; lane < 2u; lane++) { + const auto component = pair * 2u + lane; + if (((exp.en >> component) & 1u) == 0u) { + continue; + } + raw[component] = state.builder.AllocateId(); + state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, raw[component], + packed, ConstantU32(state, lane * 16u), + ConstantU32(state, 16)}); + } + } + } else { + for (uint32_t component = 0; component < 4u; component++) { + if (((exp.en >> component) & 1u) != 0u) { + raw[component] = ExportRawComponent(ctx, data, component); + } + } + } + if (uint_output) { + const auto vector = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeConstruct, state.vec4_uint_type, vector, raw[0], raw[1], raw[2], raw[3]}); + return vector; + } + uint32_t f32[4] {}; + for (uint32_t component = 0; component < 4u; component++) { + f32[component] = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.float_type, f32[component], raw[component]}); + } + const auto vector = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]}); + return vector; +} + +void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto& exp = ctx.Export(inst); + const auto exec = ctx.Arg(inst, 1); + if (state.stage == ShaderType::Pixel && exp.vm && state.needs_pixel_valid_mask && + state.pixel_valid_mask_variable != 0) { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpSelect, state.uint_type, value, exec, ConstantU32(state, 1), ConstantU32(state, 0)}); + state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value}); + } + if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive || + exp.en == 0u) { + return; + } + EmitIfCondition(state, exec, [&]() { + const auto data = ctx.Arg(inst, 0); + if (exp.kind == IR::ExportTargetKind::MrtZ) { + if ((exp.en & 1u) != 0u && state.depth_variable != 0) { + const auto raw = ExportRawComponent(ctx, data, 0); + const auto f32 = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.float_type, f32, raw}); + state.builder.AddFunction({OpStore, state.depth_variable, f32}); + } + if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) { + const auto raw = ExportRawComponent(ctx, data, 2); + const auto value = state.builder.AllocateId(); + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpBitcast, state.int_type, value, raw}); + state.builder.AddFunction({OpAccessChain, state.ptr_output_int, pointer, + state.sample_mask_variable, ConstantU32(state, 0)}); + state.builder.AddFunction({OpStore, pointer, value}); + } + return; + } + const auto variable = OutputVariableForExport(state, exp); + if (variable == 0) { + return; + } + const bool uint_output = MrtUsesUint(state, exp); + const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type; + auto value = ExportVector(ctx, data, exp, uint_output); + if (exp.kind == IR::ExportTargetKind::Mrt && state.pixel_input_info != nullptr && + exp.index < state.pixel_input_info->target_export_mapping.size()) { + const auto mapping = state.pixel_input_info->target_export_mapping[exp.index]; + if (!mapping.IsIdentity()) { + const auto mapped = state.builder.AllocateId(); + state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value, + mapping.Map(0), mapping.Map(1), mapping.Map(2), + mapping.Map(3)}); + value = mapped; + } + } + if (exp.kind == IR::ExportTargetKind::Position) { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer, + variable, ConstantU32(state, 0)}); + state.builder.AddFunction({OpStore, pointer, value}); + } else { + state.builder.AddFunction({OpStore, variable, value}); + } + }); +} + +} // namespace + +bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + switch (inst.GetOpcode()) { + case IR::ValueOpcode::Identity: + case IR::ValueOpcode::ConditionRef: ctx.Define(inst, ctx.Arg(inst, 0)); return true; + case IR::ValueOpcode::SetScalarRegister: + StoreRegisterMirror( + ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())}, + ctx.Arg(inst, 1)); + return true; + case IR::ValueOpcode::SetThreadBitScalarRegister: + StoreRegisterMirror( + ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())}, + BoolAsU32(ctx, ctx.Arg(inst, 1))); + return true; + case IR::ValueOpcode::SetVectorRegister: + StoreRegisterMirror( + ctx, {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())}, + ctx.Arg(inst, 1)); + return true; + case IR::ValueOpcode::SetScc: ctx.Arg(inst, 0); return true; + case IR::ValueOpcode::SetExec: ctx.Arg(inst, 0); return true; + case IR::ValueOpcode::SetExecLo: + case IR::ValueOpcode::SetExecHi: + if (!IsPrologue(ctx)) { + StoreRegisterMirror(ctx, + {IR::RegisterFile::Exec, + inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u}, + ctx.Arg(inst, 0)); + } + return true; + case IR::ValueOpcode::SetVcc: ctx.Arg(inst, 0); return true; + case IR::ValueOpcode::SetVccLo: + case IR::ValueOpcode::SetVccHi: + if (!IsPrologue(ctx)) { + StoreRegisterMirror(ctx, + {IR::RegisterFile::Vcc, + inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u}, + ctx.Arg(inst, 0)); + } + return true; + case IR::ValueOpcode::SetM0: + StoreRegisterMirror(ctx, {IR::RegisterFile::M0, 0}, ctx.Arg(inst, 0)); + return true; + case IR::ValueOpcode::GetScalarRegister: + case IR::ValueOpcode::GetThreadBitScalarRegister: + case IR::ValueOpcode::GetVectorRegister: + case IR::ValueOpcode::GetScc: + case IR::ValueOpcode::GetExec: + case IR::ValueOpcode::GetExecLo: + case IR::ValueOpcode::GetExecHi: + case IR::ValueOpcode::GetVcc: + case IR::ValueOpcode::GetVccLo: + case IR::ValueOpcode::GetVccHi: + case IR::ValueOpcode::GetM0: return true; + case IR::ValueOpcode::Void: + case IR::ValueOpcode::Reference: + case IR::ValueOpcode::ReferenceU32: + case IR::ValueOpcode::Prologue: + case IR::ValueOpcode::Epilogue: + case IR::ValueOpcode::ControlNop: + case IR::ValueOpcode::Waitcnt: + case IR::ValueOpcode::Sendmsg: + case IR::ValueOpcode::TtraceData: + case IR::ValueOpcode::InstPrefetch: return true; + case IR::ValueOpcode::Barrier: { + const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; + state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup), + ConstantU32(state, ScopeWorkgroup), + ConstantU32(state, semantics)}); + return true; + } + case IR::ValueOpcode::GetUserData: { + const auto reg = inst.Arg(0).ScalarRegister(); + uint32_t dword = 0; + if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) { + ctx.Define(inst, ConstantU32(state, 0)); + } else { + ctx.Define(inst, EmitShaderDataDwordLoad(state, dword)); + } + return true; + } + case IR::ValueOpcode::GetBuiltin: + ctx.Define(inst, EmitBuiltinU32(ctx, static_cast(inst.Arg(0).U32()), + inst.Arg(1).U32())); + return true; + case IR::ValueOpcode::UndefU1: + case IR::ValueOpcode::UndefU8: + case IR::ValueOpcode::UndefU16: + case IR::ValueOpcode::UndefU32: + case IR::ValueOpcode::UndefU64: + state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)}); + return true; + case IR::ValueOpcode::DppMoveU32: { + const auto flags = inst.Flags(); + IR::Operand operand; + operand.dpp = true; + operand.dpp_ctrl = flags.control; + operand.dpp_row_mask = flags.row_mask; + operand.dpp_bank_mask = flags.bank_mask; + operand.dpp_fetch_inactive = flags.fetch_inactive; + operand.dpp_bound_ctrl = flags.bound_control; + const auto target = EmitDppTargetLane(state, operand.dpp_ctrl); + const auto shuffled = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled, + ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), + target.lane}); + if (operand.dpp_fetch_inactive) { + ctx.Define(inst, shuffled); + return true; + } + const auto ballot = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)}); + const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane); + const auto can_fetch = state.builder.AllocateId(); + state.builder.AddFunction( + {OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active}); + ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)}); + return true; + } + case IR::ValueOpcode::DppUpdateU32: { + const auto flags = inst.Flags(); + const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2)); + ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; + } + case IR::ValueOpcode::WqmMask: + ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0))); + return true; + case IR::ValueOpcode::LaneId: + ctx.Define(inst, EmitSubgroupLocalInvocationId(state)); + return true; + case IR::ValueOpcode::Ballot: + if (state.per_invocation_masks) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, + BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0), + ConstantU32(state, 0), ConstantU32(state, 0)}); + ctx.Define(inst, result); + } else { + ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4, + {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)}); + } + return true; + case IR::ValueOpcode::ReadFirstLane: { + const auto ballot = state.builder.AllocateId(); + const auto lane = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)}); + state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, lane, + ConstantU32(state, ScopeSubgroup), ballot}); + ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32, + {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane}); + return true; + } + case IR::ValueOpcode::ReadLane: + ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32, + {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); + return true; + case IR::ValueOpcode::WriteLane: { + const auto hit = state.builder.AllocateId(); + state.builder.AddFunction({OpIEqual, state.bool_type, hit, + EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)}); + ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)}); + return true; + } + case IR::ValueOpcode::Permlane16U32: { + const auto flags = inst.Flags(); + const auto subid = EmitSubgroupLocalInvocationId(state); + const auto row = state.builder.AllocateId(); + const auto row_value = state.builder.AllocateId(); + const auto lane = state.builder.AllocateId(); + const auto lane8 = state.builder.AllocateId(); + const auto shift = state.builder.AllocateId(); + const auto upper = state.builder.AllocateId(); + const auto selected = state.builder.AllocateId(); + const auto shifted = state.builder.AllocateId(); + const auto index = state.builder.AllocateId(); + const auto target = state.builder.AllocateId(); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)}); + if (flags.x16) { + state.builder.AddFunction( + {OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)}); + } else { + state.builder.AddFunction({OpCopyObject, state.uint_type, row_value, row}); + } + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)}); + state.builder.AddFunction( + {OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)}); + state.builder.AddFunction( + {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)}); + state.builder.AddFunction( + {OpSelect, state.uint_type, selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)}); + state.builder.AddFunction( + {OpShiftRightLogical, state.uint_type, shifted, selected, shift}); + state.builder.AddFunction( + {OpBitwiseAnd, state.uint_type, index, shifted, ConstantU32(state, 15)}); + state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index}); + const auto shuffled = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled, + ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), + target}); + uint32_t result = shuffled; + if (!flags.fetch_inactive) { + const auto source_exec = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec, + ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3), + target}); + result = state.builder.AllocateId(); + state.builder.AddFunction({OpSelect, state.uint_type, result, source_exec, shuffled, + ConstantU32(state, 0)}); + } + ctx.Define(inst, result); + return true; + } + case IR::ValueOpcode::GetAttribute: + ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32())); + return true; + case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true; + case IR::ValueOpcode::GetShaderBase: + // Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The + // runtime descriptor evaluator supplies the mapped shader base for host-side planning. + ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0}))); + return true; + case IR::ValueOpcode::GetSrtResource: + case IR::ValueOpcode::GetBufferResource: + case IR::ValueOpcode::GetAddressResource: + case IR::ValueOpcode::GetImageResource: + case IR::ValueOpcode::GetSamplerResource: + case IR::ValueOpcode::GetLdsResource: + case IR::ValueOpcode::GetGdsResource: + case IR::ValueOpcode::MakeImageAddress: return true; + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp new file mode 100644 index 000000000..d5f64d812 --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueImage.cpp @@ -0,0 +1,516 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, type, result, value}); + return result; +} + +uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, type, result, lhs, rhs}); + return result; +} + +bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) { + return (mem.image_sample_flags & flag) != 0u; +} + +bool UsesA16(const IR::MemoryInfo& mem) { + return HasFlag(mem, Decoder::ImageSampleFlagA16); +} + +bool ComponentUsesA16(const IR::MemoryInfo& mem, uint32_t component) { + if (!UsesA16(mem)) return false; + uint32_t cursor = 0; + if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) { + if (component == cursor) return false; + cursor++; + } + if (HasFlag(mem, Decoder::ImageSampleFlagBias)) { + if (component == cursor) return true; + cursor++; + } + if (HasFlag(mem, Decoder::ImageSampleFlagCompare) && component == cursor) return false; + return true; +} + +uint32_t HalfComponent(const IR::MemoryInfo& mem, uint32_t component) { + if (!UsesA16(mem)) return component * 2u; + uint32_t half = 0; + for (uint32_t index = 0; index < component; index++) { + const auto width = ComponentUsesA16(mem, index) ? 1u : 2u; + if (width == 2u && (half & 1u) != 0u) half++; + half += width; + } + if (!ComponentUsesA16(mem, component) && (half & 1u) != 0u) half++; + return half; +} + +uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address, + uint32_t component) { + const auto half = HalfComponent(mem, component); + const auto packed = half / 2u; + if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0); + auto value = ctx.Def(address.Arg(packed)); + if (ComponentUsesA16(mem, component)) { + if ((half & 1u) != 0u) { + value = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, value, + ConstantU32(ctx.state, 16)); + } + value = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, value, + ConstantU32(ctx.state, 0xffffu)); + } + return value; +} + +uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address, + uint32_t component) { + const auto value = AddressU32(ctx, mem, address, component); + return ComponentUsesA16(mem, component) + ? EmitF16BitsToF32(ctx.state, value) + : Unary(ctx.state, OpBitcast, ctx.state.float_type, value); +} + +ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) { + ImageSampleLayout layout; + uint32_t cursor = 0; + if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++; + if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++; + if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++; + if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) { + layout.grad_x = cursor; + cursor += ImageViewSpatialComponents(view); + layout.grad_y = cursor; + cursor += ImageViewSpatialComponents(view); + } + layout.coord = cursor; + cursor += ImageViewCoordinateComponents(view); + if (HasFlag(mem, Decoder::ImageSampleFlagLod)) layout.lod = cursor++; + return layout; +} + +uint32_t ZeroF32(EmitterState& state) { + return ConstantF32(state, 0); +} + +uint32_t ZeroU32x4(EmitterState& state) { + const auto zero = ConstantU32(state, 0); + const auto value = state.builder.AllocateId(); + state.builder.AddType( + {OpConstantComposite, state.vec4_uint_type, value, zero, zero, zero, zero}); + return value; +} + +uint32_t CubeAxis(EmitterState& state, uint32_t value) { + return Binary(state, OpFSub, state.float_type, value, ConstantF32(state, 0x3f800000u)); +} + +uint32_t CubeLayer(EmitterState& state, uint32_t value) { + const auto guest = state.builder.AllocateId(); + state.builder.AddFunction({OpConvertFToU, state.uint_type, guest, value}); + const auto padding = + Binary(state, OpShiftLeftLogical, state.uint_type, + Binary(state, OpShiftRightLogical, state.uint_type, guest, ConstantU32(state, 3)), + ConstantU32(state, 1)); + const auto host = Binary(state, OpISub, state.uint_type, guest, padding); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpConvertUToF, state.float_type, result, host}); + return result; +} + +uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address, + uint32_t first, uint32_t components) { + auto x = AddressF32(ctx, mem, address, first); + if (components == 1u) return x; + auto y = mem.image_address_components > first + 1u ? AddressF32(ctx, mem, address, first + 1u) + : ZeroF32(ctx.state); + if (mem.image_cube) { + x = CubeAxis(ctx.state, x); + y = CubeAxis(ctx.state, y); + } + const auto result = ctx.state.builder.AllocateId(); + if (components == 3u) { + auto z = mem.image_address_components > first + 2u + ? AddressF32(ctx, mem, address, first + 2u) + : ZeroF32(ctx.state); + if (mem.image_cube) z = CubeLayer(ctx.state, z); + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec3_float_type, result, x, y, z}); + } else { + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec2_float_type, result, x, y}); + } + return result; +} + +uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address, + ImageViewKind view) { + const auto components = ImageViewCoordinateComponents(view); + const auto x = AddressU32(ctx, mem, address, 0); + if (components == 1u) return x; + const auto y = mem.image_address_components > 1u ? AddressU32(ctx, mem, address, 1) + : ConstantU32(ctx.state, 0); + const auto result = ctx.state.builder.AllocateId(); + if (components == 3u) { + const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2) + : ConstantU32(ctx.state, 0); + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec3_uint_type, result, x, y, z}); + } else { + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec2_uint_type, result, x, y}); + } + return result; +} + +uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address, + ImageViewKind view) { + const auto component = ImageViewCoordinateComponents(view); + return mem.image_has_mip && mem.image_address_components > component + ? AddressU32(ctx, mem, address, component) + : ConstantU32(ctx.state, 0); +} + +uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) { + return Unary(ctx.state, OpBitcast, ctx.state.uint_type, value); +} + +uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) { + uint32_t component[4] {}; + for (uint32_t index = 0; index < 4u; index++) { + if (dref) { + component[index] = index == 0u ? FloatBits(ctx, value) : ConstantU32(ctx.state, 0); + continue; + } + const auto scalar = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpCompositeExtract, + integer ? ctx.state.uint_type : ctx.state.float_type, scalar, + value, index}); + component[index] = integer ? scalar : FloatBits(ctx, scalar); + } + const auto result = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, result, + component[0], component[1], component[2], component[3]}); + return result; +} + +uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + const IR::Inst& address) { + const auto pc = inst.Flags().pc; + const auto view = SampledImageViewKind(ctx.state, mem, pc); + const auto image = LoadSampledImageDescriptor(ctx.state, mem, pc, view); + const auto size = ctx.state.builder.AllocateId(); + if (ImageSpirvMultisampled(view) != 0u) { + ctx.state.builder.AddFunction( + {OpImageQuerySize, ImageViewSizeType(ctx.state, view), size, image}); + } else { + ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, view), + size, image, AddressU32(ctx, mem, address, 0)}); + } + const auto components = ImageViewCoordinateComponents(view); + uint32_t result[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0), + ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)}; + if (components == 1u) { + result[0] = size; + } else { + for (uint32_t index = 0; index < components; index++) { + result[index] = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpCompositeExtract, ctx.state.uint_type, result[index], size, index}); + } + } + if (ImageSpirvMultisampled(view) == 0u) { + result[3] = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpImageQueryLevels, ctx.state.uint_type, result[3], image}); + } + const auto vector = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, vector, + result[0], result[1], result[2], result[3]}); + return vector; +} + +uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address, + const ImageSampleLayout& layout, ImageViewKind view) { + const auto components = ImageViewSpatialComponents(view); + const auto zero = ConstantI32(ctx.state, 0); + if (layout.offset == NoImageComponent || mem.image_address_components <= layout.offset) { + if (components == 1u) return zero; + const auto result = ctx.state.builder.AllocateId(); + if (components == 3u) { + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec3_int_type, result, zero, zero, zero}); + } else { + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec2_int_type, result, zero, zero}); + } + return result; + } + const auto packed = Unary(ctx.state, OpBitcast, ctx.state.int_type, + AddressU32(ctx, mem, address, layout.offset)); + uint32_t values[3] = {zero, zero, zero}; + for (uint32_t index = 0; index < components; index++) { + values[index] = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpBitFieldSExtract, ctx.state.int_type, values[index], + packed, ConstantU32(ctx.state, index * 8u), + ConstantU32(ctx.state, 6)}); + } + if (components == 1u) return values[0]; + const auto result = ctx.state.builder.AllocateId(); + if (components == 3u) { + ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec3_int_type, result, + values[0], values[1], values[2]}); + } else { + ctx.state.builder.AddFunction( + {OpCompositeConstruct, ctx.state.vec2_int_type, result, values[0], values[1]}); + } + return result; +} + +uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) { + const auto components = ImageViewSpatialComponents(view); + const auto array_type = state.builder.AllocateId(); + const auto count = ConstantU32(state, 4); + const auto element_type = components == 1u ? state.int_type : state.vec2_int_type; + state.builder.AddType({OpTypeArray, array_type, element_type, count}); + uint32_t offsets[4] {}; + for (uint32_t index = 0; index < 4u; index++) { + const auto x = ConstantI32(state, static_cast(index) - 1); + if (components == 1u) { + offsets[index] = x; + } else { + offsets[index] = state.builder.AllocateId(); + state.builder.AddType({OpConstantComposite, state.vec2_int_type, offsets[index], x, + ConstantI32(state, 0)}); + } + } + const auto value = state.builder.AllocateId(); + state.builder.AddType( + {OpConstantComposite, array_type, value, offsets[0], offsets[1], offsets[2], offsets[3]}); + return value; +} + +uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) { + for (uint32_t source = 0; source < 4u; source++) { + if (((swizzle >> (source * 3u)) & 7u) == 4u + component) return source; + } + return UINT32_MAX; +} + +uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) { + const auto swizzle = ctx.state.program.info.images[mem.resource].storage_swizzle; + uint32_t values[4] {}; + const auto dmask = mem.dmask != 0u ? mem.dmask : 1u; + for (uint32_t component = 0; component < 4u; component++) { + const auto source = InverseSwizzle(swizzle, component); + uint32_t raw = ConstantU32(ctx.state, 0); + if (source < 4u && ((dmask >> source) & 1u) != 0u) { + const auto packed_index = DmaskComponentIndex(dmask, source); + raw = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpCompositeExtract, ctx.state.uint_type, raw, data, packed_index}); + } + values[component] = integer ? raw : Unary(ctx.state, OpBitcast, ctx.state.float_type, raw); + } + const auto texel = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpCompositeConstruct, + integer ? ctx.state.vec4_uint_type : ctx.state.vec4_float_type, + texel, values[0], values[1], values[2], values[3]}); + return texel; +} + +uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) { + switch (opcode) { + case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd; + case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin; + case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax; + case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd; + case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr; + case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor; + default: return 0; + } +} + +} // namespace + +bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) { + const auto op = inst.GetOpcode(); + if (op < IR::ValueOpcode::ImageQueryDimensions || op > IR::ValueOpcode::ImageAtomicXor32) { + return false; + } + auto& state = ctx.state; + const auto& mem = ctx.Memory(inst); + const auto pc = inst.Flags().pc; + const auto image_arg = inst.Arg(0); + ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource); + const bool sampled = op == IR::ValueOpcode::ImageQueryLod || + op == IR::ValueOpcode::ImageSampleRaw || + op == IR::ValueOpcode::ImageGatherRaw; + const auto* address = ctx.ImageAddress(inst.Arg(sampled ? 2 : 1)); + if (address == nullptr) return true; + if (op == IR::ValueOpcode::ImageQueryDimensions) { + ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address)); + return true; + } + if (op == IR::ValueOpcode::ImageQueryLod) { + const auto view = SampledImageViewKind(state, mem, pc); + const auto sampled = MakeSampledImage(state, mem, pc, view); + const auto lod = state.builder.AllocateId(); + state.builder.AddFunction( + {OpImageQueryLod, state.vec2_float_type, lod, sampled, + CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))}); + uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), + ConstantU32(state, 0)}; + for (uint32_t index = 0; index < 2u; index++) { + const auto component = state.builder.AllocateId(); + state.builder.AddFunction( + {OpCompositeExtract, state.float_type, component, lod, index}); + values[index] = FloatBits(ctx, component); + } + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, values[0], + values[1], values[2], values[3]}); + ctx.Define(inst, result); + return true; + } + if (op == IR::ValueOpcode::ImageRead) { + const auto view = SampledImageViewKind(state, mem, pc); + const bool integer = mem.kind == IR::ResourceKind::ImageUint; + const auto condition = ctx.Arg(inst, 2); + ctx.Define( + inst, + EmitValueOrDefaultIfCondition( + state, condition, state.vec4_uint_type, ZeroU32x4(state), [&]() { + const auto image = LoadSampledImageDescriptor(state, mem, pc, view); + const auto color = state.builder.AllocateId(); + const auto coord = CoordU32(ctx, mem, *address, view); + if (ImageSpirvMultisampled(view) != 0u) { + state.builder.AddFunction( + {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type, + color, image, coord, ImageOperandsSampleMask, + AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))}); + } else { + state.builder.AddFunction( + {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type, + color, image, coord, ImageOperandsLodMask, + LodU32(ctx, mem, *address, view)}); + } + return ResultVector(ctx, color, integer, false); + })); + return true; + } + if (op == IR::ValueOpcode::ImageWrite) { + const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint; + const auto view = StorageImageViewKind(state, mem, uint_image, pc); + EmitIfCondition(state, ctx.Arg(inst, 3), [&]() { + state.builder.AddFunction( + {OpImageWrite, + LoadStorageImageDescriptor(state, mem.resource, uint_image, pc, view), + CoordU32(ctx, mem, *address, view), + StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image)}); + }); + return true; + } + if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) { + const auto view = SampledImageViewKind(state, mem, pc); + const auto layout = Layout(mem, view); + const bool integer = mem.kind == IR::ResourceKind::ImageUint; + const bool dref = HasFlag(mem, Decoder::ImageSampleFlagCompare); + const auto sampled = MakeSampledImage(state, mem, pc, view); + const auto coord = + CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view)); + const auto sample = state.builder.AllocateId(); + if (op == IR::ValueOpcode::ImageGatherRaw) { + std::vector words = + dref ? std::vector {OpImageDrefGather, + state.vec4_float_type, + sample, + sampled, + coord, + layout.dref != NoImageComponent + ? AddressF32(ctx, mem, *address, layout.dref) + : ZeroF32(state)} + : std::vector { + OpImageGather, integer ? state.vec4_uint_type : state.vec4_float_type, + sample, sampled, + coord, ConstantU32(state, ImageGatherComponent(mem.dmask))}; + if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) { + words.push_back(ImageOperandsConstOffsetsMask); + words.push_back(HorizontalOffsets(state, view)); + } else if (layout.offset != NoImageComponent) { + words.push_back(ImageOperandsOffsetMask); + words.push_back(PackedOffset(ctx, mem, *address, layout, view)); + } + state.builder.AddFunction(words); + ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false)); + return true; + } + const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) || + HasFlag(mem, Decoder::ImageSampleFlagLod) || + HasFlag(mem, Decoder::ImageSampleFlagLevelZero) || + state.stage != ShaderType::Pixel; + const auto opcode = explicit_lod + ? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod) + : (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod); + const auto result_type = + dref ? state.float_type : (integer ? state.vec4_uint_type : state.vec4_float_type); + std::vector words {opcode, result_type, sample, sampled, coord}; + if (dref) { + words.push_back(layout.dref != NoImageComponent + ? AddressF32(ctx, mem, *address, layout.dref) + : ZeroF32(state)); + } + uint32_t operand_mask = 0; + std::vector operands; + if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) { + operand_mask |= ImageOperandsGradMask; + operands.push_back( + CoordF32(ctx, mem, *address, layout.grad_x, ImageViewSpatialComponents(view))); + operands.push_back( + CoordF32(ctx, mem, *address, layout.grad_y, ImageViewSpatialComponents(view))); + } else if (explicit_lod) { + operand_mask |= ImageOperandsLodMask; + operands.push_back(HasFlag(mem, Decoder::ImageSampleFlagLod) && + layout.lod != NoImageComponent + ? AddressF32(ctx, mem, *address, layout.lod) + : ZeroF32(state)); + } else if (layout.bias != NoImageComponent) { + operand_mask |= ImageOperandsBiasMask; + operands.push_back(AddressF32(ctx, mem, *address, layout.bias)); + } + if (operand_mask != 0u) { + words.push_back(operand_mask); + words.insert(words.end(), operands.begin(), operands.end()); + } + state.builder.AddFunction(words); + ctx.Define(inst, ResultVector(ctx, sample, integer, dref)); + return true; + } + const auto atomic_opcode = ImageAtomicOpcode(op); + if (atomic_opcode != 0u) { + const auto view = StorageImageViewKind(state, mem, true, pc); + ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction( + {OpImageTexelPointer, state.ptr_image_uint, pointer, + StorageImageDescriptorPointer(state, mem.resource, true, pc, view), + CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)}); + const auto old = state.builder.AllocateId(); + state.builder.AddFunction({atomic_opcode, state.uint_type, old, pointer, + ConstantU32(state, ScopeDevice), + ConstantU32(state, MemorySemanticsNone), + ctx.Arg(inst, 2)}); + EmitDeviceAtomicMemoryBarrier(state); + return old; + })); + return true; + } + return false; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp new file mode 100644 index 000000000..356b14a6c --- /dev/null +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValueMemory.cpp @@ -0,0 +1,713 @@ +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { +namespace { + +uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, type, result, value}); + return result; +} + +uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({opcode, type, result, lhs, rhs}); + return result; +} + +uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value, + uint32_t false_value) { + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value}); + return result; +} + +uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) { + if (lhs == 0) return rhs; + if (rhs == 0) return lhs; + return Binary(state, OpLogicalAnd, state.bool_type, lhs, rhs); +} + +uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + uint32_t index, uint32_t offset, uint32_t soffset) { + auto& state = ctx.state; + const auto packed = + ConstantU32(state, StorageBufferPackedStride(state, mem, inst.Flags().pc)); + if (mem.formatted && !mem.typed && inst.GetOpcode() == IR::ValueOpcode::StoreBufferU32) { + const auto add_tid_bit = Binary( + state, OpBitwiseAnd, state.uint_type, + Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 20)), + ConstantU32(state, 1)); + const auto add_tid = + Binary(state, OpINotEqual, state.bool_type, add_tid_bit, ConstantU32(state, 0)); + const auto lane = Binary(state, OpBitwiseAnd, state.uint_type, + EmitLocalInvocationIndex(state), ConstantU32(state, 63)); + index = Select(state, state.uint_type, add_tid, + Binary(state, OpIAdd, state.uint_type, index, lane), index); + } + const auto stride = + Binary(state, OpBitwiseAnd, state.uint_type, packed, ConstantU32(state, 0x3fffu)); + const auto swizzle = + Binary(state, OpBitwiseAnd, state.uint_type, + Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 14)), + ConstantU32(state, 1)); + const auto use_swizzle = + Binary(state, OpLogicalAnd, state.bool_type, + Binary(state, OpINotEqual, state.bool_type, stride, ConstantU32(state, 0)), + Binary(state, OpINotEqual, state.bool_type, swizzle, ConstantU32(state, 0))); + const auto stride_enum = + Binary(state, OpBitwiseAnd, state.uint_type, + Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 16)), + ConstantU32(state, 3)); + const auto index_shift = + Binary(state, OpIAdd, state.uint_type, stride_enum, ConstantU32(state, 3)); + const auto index_stride = + Binary(state, OpShiftLeftLogical, state.uint_type, ConstantU32(state, 8), stride_enum); + const auto index_mask = + Binary(state, OpISub, state.uint_type, index_stride, ConstantU32(state, 1)); + offset = Binary(state, OpIAdd, state.uint_type, offset, ConstantU32(state, mem.offset)); + const auto linear = Binary(state, OpIAdd, state.uint_type, + Binary(state, OpIMul, state.uint_type, index, stride), offset); + const auto index_msb = Binary(state, OpShiftRightLogical, state.uint_type, index, index_shift); + const auto index_lsb = Binary(state, OpBitwiseAnd, state.uint_type, index, index_mask); + const auto offset_msb = + Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, ~3u)); + const auto offset_lsb = + Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, 3)); + const auto msb = + Binary(state, OpIMul, state.uint_type, + Binary(state, OpIAdd, state.uint_type, + Binary(state, OpIMul, state.uint_type, index_msb, stride), offset_msb), + index_stride); + const auto lsb = + Binary(state, OpIAdd, state.uint_type, + Binary(state, OpShiftLeftLogical, state.uint_type, index_lsb, ConstantU32(state, 2)), + offset_lsb); + const auto address = Select(state, state.uint_type, use_swizzle, + Binary(state, OpIAdd, state.uint_type, msb, lsb), linear); + return Binary(state, OpIAdd, state.uint_type, address, soffset); +} + +uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low, + uint32_t add_high, uint32_t& out_high) { + const auto result = Binary(state, OpIAdd, state.uint_type, low, add_low); + const auto carry = Binary(state, OpULessThan, state.bool_type, result, low); + out_high = Binary( + state, OpIAdd, state.uint_type, Binary(state, OpIAdd, state.uint_type, high, add_high), + Select(state, state.uint_type, carry, ConstantU32(state, 1), ConstantU32(state, 0))); + return result; +} + +uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + uint32_t low, uint32_t high) { + auto& state = ctx.state; + const auto immediate = static_cast(mem.offset); + const auto immediate_low = ConstantU32(state, static_cast(immediate)); + const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u); + const auto base = state.program.info.addresses[mem.resource].specialized_base; + if (mem.kind == IR::ResourceKind::Scratch) { + low = AddU64Low(state, low, high, immediate_low, immediate_high, high); + const auto valid = Binary(state, OpIEqual, state.bool_type, high, ConstantU32(state, 0)); + return Select(state, state.uint_type, valid, low, ConstantU32(state, UINT32_MAX)); + } + if (mem.kind == IR::ResourceKind::Flat || + (mem.kind == IR::ResourceKind::Global && + state.program.info.addresses[mem.resource].unbased)) { + low = AddU64Low(state, low, high, immediate_low, immediate_high, high); + const auto base_low = ConstantU32(state, static_cast(base)); + const auto base_high = ConstantU32(state, static_cast(base >> 32u)); + const auto relative = Binary(state, OpISub, state.uint_type, low, base_low); + const auto borrow = Binary(state, OpULessThan, state.bool_type, low, base_low); + const auto relative_high = Binary( + state, OpISub, state.uint_type, Binary(state, OpISub, state.uint_type, high, base_high), + Select(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0))); + const auto valid = + Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0)); + return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX)); + } + const auto initial = base + static_cast(static_cast(immediate)); + auto relative_high = ConstantU32(state, static_cast(initial >> 32u)); + const auto relative = AddU64Low(state, ConstantU32(state, static_cast(initial)), + relative_high, low, ConstantU32(state, 0), relative_high); + const auto valid = + Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0)); + return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX)); +} + +uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + if (mem.kind == IR::ResourceKind::Buffer || mem.kind == IR::ResourceKind::ScalarBuffer) { + if (mem.kind == IR::ResourceKind::ScalarBuffer) { + return Binary(ctx.state, OpIAdd, ctx.state.uint_type, ctx.Arg(inst, 1), + ConstantU32(ctx.state, mem.offset)); + } + return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2), + ctx.Arg(inst, 3)); + } + if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { + return ctx.Arg(inst, 1); + } + return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2)); +} + +uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + return Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, ByteAddress(ctx, inst, mem), + ConstantU32(ctx.state, 2)); +} + +uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) { + if (mem.kind == IR::ResourceKind::Lds) { + return 0; + } + if (mem.kind == IR::ResourceKind::Gds) { + return EmitGdsElementInBounds(ctx.state, index); + } + return EmitStorageBufferElementInBounds(ctx.state, mem, index, pc); +} + +uint32_t Pointer(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) { + return EmitMemoryElementPointer(ctx.state, mem, index, pc); +} + +uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, + uint32_t extra_offset = 0) { + mem.offset += extra_offset; + const auto pc = inst.Flags().pc; + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto index = DwordIndex(ctx, inst, mem); + return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + const auto value = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, ctx.state.uint_type, value, Pointer(ctx, mem, index, pc)}); + return value; + }); + }); +} + +uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits, + bool sign_extend, uint32_t extra_offset = 0) { + mem.offset += extra_offset; + const auto pc = inst.Flags().pc; + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto address = ByteAddress(ctx, inst, mem); + const auto index = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address, + ConstantU32(ctx.state, 2)); + return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + const auto word = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, ctx.state.uint_type, word, Pointer(ctx, mem, index, pc)}); + const auto byte = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address, + ConstantU32(ctx.state, 3)); + const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, byte, + ConstantU32(ctx.state, 3)); + const auto value = + Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, + Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, word, shift), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); + if (!sign_extend) return value; + const auto left = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, value, + ConstantU32(ctx.state, 32u - bits)); + return Binary(ctx.state, OpShiftRightArithmetic, ctx.state.uint_type, left, + ConstantU32(ctx.state, 32u - bits)); + }); + }); +} + +Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& inst, + const IR::MemoryInfo& mem) { + return mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format) + : StorageBufferFormat(ctx.state, mem, inst.Flags().pc); +} + +IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format, + uint32_t component) { + if (!mem.typed && component != 0u) { + const auto component_slot = component * 4u; + mem.offset = mem.offset >= component_slot ? mem.offset - component_slot : 0u; + mem.offset += Format::GetFormatComponentByteOffset(format, component); + } + return mem; +} + +uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + const auto format = BufferFormat(ctx, inst, mem); + if (!Format::IsKnownFormat(format)) return LoadWord(ctx, inst, mem); + const auto info = Format::GetFormatInfo(format); + const auto component = mem.component_index; + if (component >= info.component_count) return ConstantU32(ctx.state, 0); + uint32_t raw = 0; + const auto bits = info.component_bits[component]; + if (info.packed_bitfield) { + raw = LoadWord(ctx, inst, mem); + const auto type = + IsSignedFormatComponent(info.type) ? ctx.state.int_type : ctx.state.uint_type; + const auto source = + type == ctx.state.int_type ? Unary(ctx.state, OpBitcast, type, raw) : raw; + const auto extracted = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type, + extracted, source, ConstantU32(ctx.state, info.component_bit_offset[component]), + ConstantU32(ctx.state, bits)}); + raw = type == ctx.state.int_type + ? Unary(ctx.state, OpBitcast, ctx.state.uint_type, extracted) + : extracted; + } else if (bits == 32u) { + raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component)); + } else { + raw = LoadSubword(ctx, inst, RebaseFormattedComponent(mem, format, component), bits, + IsSignedFormatComponent(info.type)); + } + return NormalizeFormatComponent(ctx.state, info, component, raw); +} + +template +uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) { + const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; + const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory + : MemorySemanticsUniformMemory; + const auto preheader = state.builder.AllocateId(); + const auto header = state.builder.AllocateId(); + const auto cont = state.builder.AllocateId(); + const auto merge = state.builder.AllocateId(); + const auto initial = state.builder.AllocateId(); + const auto observed = state.builder.AllocateId(); + const auto exchanged = state.builder.AllocateId(); + state.builder.AddFunction({OpBranch, preheader}); + state.builder.AddFunction({OpLabel, preheader}); + state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer, + ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)}); + state.builder.AddFunction({OpBranch, header}); + state.builder.AddFunction({OpLabel, header}); + state.builder.AddFunction( + {OpPhi, state.uint_type, observed, initial, preheader, exchanged, cont}); + const auto next = desired(observed); + state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer, + ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone), + ConstantU32(state, MemorySemanticsNone), next, observed}); + const auto success = Binary(state, OpIEqual, state.bool_type, exchanged, observed); + state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone}); + state.builder.AddFunction({OpBranchConditional, success, merge, cont}); + state.builder.AddFunction({OpLabel, cont}); + state.builder.AddFunction({OpBranch, header}); + state.builder.AddFunction({OpLabel, merge}); + state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope), + ConstantU32(state, MemorySemanticsAcquireRelease | memory)}); + return observed; +} + +void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits, + uint32_t extra_offset = 0) { + mem.offset += extra_offset; + const auto pc = inst.Flags().pc; + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto address = ByteAddress(ctx, inst, mem); + const auto index = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address, + ConstantU32(ctx.state, 2)); + EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + const auto pointer = Pointer(ctx, mem, index, pc); + const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, + Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address, + ConstantU32(ctx.state, 3)), + ConstantU32(ctx.state, 3)); + const auto mask = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift); + const auto value = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, + Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, + ctx.Arg(inst, inst.NumArgs() - 2), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)), + shift); + const auto merge = [&](uint32_t old) { + return Binary(ctx.state, OpBitwiseOr, ctx.state.uint_type, + Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, old, + Unary(ctx.state, OpNot, ctx.state.uint_type, mask)), + value); + }; + if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { + AtomicUpdate(ctx.state, pointer, mem.kind, merge); + } else { + const auto old = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpLoad, ctx.state.uint_type, old, pointer}); + ctx.state.builder.AddFunction({OpStore, pointer, merge(old)}); + } + }); + }); +} + +void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, + uint32_t extra_offset = 0) { + mem.offset += extra_offset; + const auto pc = inst.Flags().pc; + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto index = DwordIndex(ctx, inst, mem); + EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + ctx.state.builder.AddFunction( + {OpStore, Pointer(ctx, mem, index, pc), ctx.Arg(inst, inst.NumArgs() - 2)}); + }); + }); +} + +void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { + const auto format = BufferFormat(ctx, inst, mem); + if (!Format::IsKnownFormat(format)) { + StoreWord(ctx, inst, mem); + return; + } + const auto info = Format::GetFormatInfo(format); + if (mem.component_index >= info.component_count) return; + const auto bits = info.component_bits[mem.component_index]; + const auto component_mem = RebaseFormattedComponent(mem, format, mem.component_index); + if (bits == 8u || bits == 16u) { + StoreSubword(ctx, inst, component_mem, bits); + } else { + StoreWord(ctx, inst, component_mem); + } +} + +uint32_t AtomicOpcode(IR::ValueOpcode opcode) { + switch (opcode) { + case IR::ValueOpcode::BufferAtomicSwap32: + case IR::ValueOpcode::SharedAtomicSwap32: + case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange; + case IR::ValueOpcode::BufferAtomicIAdd32: + case IR::ValueOpcode::SharedAtomicIAdd32: + case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd; + case IR::ValueOpcode::BufferAtomicISub32: + case IR::ValueOpcode::SharedAtomicISub32: + case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub; + case IR::ValueOpcode::BufferAtomicSMin32: + case IR::ValueOpcode::SharedAtomicSMin32: + case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin; + case IR::ValueOpcode::BufferAtomicUMin32: + case IR::ValueOpcode::SharedAtomicUMin32: + case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin; + case IR::ValueOpcode::BufferAtomicSMax32: + case IR::ValueOpcode::SharedAtomicSMax32: + case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax; + case IR::ValueOpcode::BufferAtomicUMax32: + case IR::ValueOpcode::SharedAtomicUMax32: + case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax; + case IR::ValueOpcode::BufferAtomicAnd32: + case IR::ValueOpcode::SharedAtomicAnd32: + case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd; + case IR::ValueOpcode::BufferAtomicOr32: + case IR::ValueOpcode::SharedAtomicOr32: + case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr; + case IR::ValueOpcode::BufferAtomicXor32: + case IR::ValueOpcode::SharedAtomicXor32: + case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor; + default: return 0; + } +} + +uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + bool return_value) { + const auto pc = inst.Flags().pc; + const auto result = + EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto index = DwordIndex(ctx, inst, mem); + return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + const auto value = ctx.Arg(inst, inst.NumArgs() - 2); + const auto old = ctx.state.builder.AllocateId(); + const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; + ctx.state.builder.AddFunction({AtomicOpcode(inst.GetOpcode()), ctx.state.uint_type, + old, Pointer(ctx, mem, index, pc), + ConstantU32(ctx.state, scope), + ConstantU32(ctx.state, MemorySemanticsNone), value}); + if (mem.kind == IR::ResourceKind::Lds) { + const auto semantics = + MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; + ctx.state.builder.AddFunction({OpMemoryBarrier, + ConstantU32(ctx.state, ScopeWorkgroup), + ConstantU32(ctx.state, semantics)}); + } else { + EmitDeviceAtomicMemoryBarrier(ctx.state); + } + return old; + }); + }); + return return_value ? result : ConstantU32(ctx.state, 0); +} + +uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, + bool max_value) { + struct OrderedBits { + uint32_t nan; + uint32_t zero; + uint32_t key; + }; + const auto classify = [&](uint32_t bits) { + const auto value = Unary(state, OpBitcast, state.float_type, bits); + const auto cls = EmitClassifyF32(state, value); + const auto negative = Binary( + state, OpINotEqual, state.bool_type, + Binary(state, OpBitwiseAnd, state.uint_type, bits, ConstantU32(state, 0x80000000u)), + ConstantU32(state, 0)); + const auto negative_key = Unary(state, OpNot, state.uint_type, bits); + const auto positive_key = + Binary(state, OpBitwiseXor, state.uint_type, bits, ConstantU32(state, 0x80000000u)); + return OrderedBits {cls.nan, cls.zero, + Select(state, state.uint_type, negative, negative_key, positive_key)}; + }; + const auto source_class = classify(source); + const auto old_class = classify(old); + const auto unordered = + Binary(state, OpLogicalOr, state.bool_type, + Binary(state, OpLogicalOr, state.bool_type, source_class.nan, old_class.nan), + Binary(state, OpLogicalAnd, state.bool_type, source_class.zero, old_class.zero)); + const auto ordered = Unary(state, OpLogicalNot, state.bool_type, unordered); + const auto compare = Binary(state, max_value ? OpUGreaterThan : OpULessThan, state.bool_type, + source_class.key, old_class.key); + return Select(state, state.uint_type, + Binary(state, OpLogicalAnd, state.bool_type, ordered, compare), source, old); +} + +uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + bool max_value) { + const auto pc = inst.Flags().pc; + return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto index = DwordIndex(ctx, inst, mem); + return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + const auto pointer = Pointer(ctx, mem, index, pc); + const auto source = ctx.Arg(inst, inst.NumArgs() - 2); + return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) { + return FloatAtomicReplacement(ctx.state, old, source, max_value); + }); + }); + }); +} + +uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, + bool max_value) { + const auto pc = inst.Flags().pc; + EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto index = DwordIndex(ctx, inst, mem); + EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { + ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)}); + const auto data = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, ctx.state.uint_type, data, ctx.scratch_u32_variable}); + AtomicUpdate(ctx.state, Pointer(ctx, mem, index, pc), mem.kind, [&](uint32_t old) { + const auto old_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, old); + const auto compare_f = + Unary(ctx.state, OpBitcast, ctx.state.float_type, ctx.Arg(inst, 3)); + const auto data_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, data); + const auto compare = Binary( + ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, ctx.state.bool_type, + max_value ? old_f : compare_f, max_value ? compare_f : old_f); + return Unary(ctx.state, OpBitcast, ctx.state.uint_type, + Select(ctx.state, ctx.state.float_type, compare, data_f, old_f)); + }); + }); + }); + return 0; +} + +uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) { + auto& state = ctx.state; + const auto m0 = ctx.Arg(inst, 1); + const auto base = + Binary(state, OpShiftRightLogical, state.uint_type, m0, ConstantU32(state, 16)); + const auto size = Binary(state, OpBitwiseAnd, state.uint_type, m0, ConstantU32(state, 0xffffu)); + const auto address = + Binary(state, OpIAdd, state.uint_type, base, ConstantU32(state, ctx.Memory(inst).offset)); + const auto index = + Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2)); + const auto exec = ctx.Arg(inst, 2); + const auto ballot = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot, + ConstantU32(state, ScopeSubgroup), exec}); + const auto low = state.builder.AllocateId(); + const auto high = state.builder.AllocateId(); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0}); + state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1}); + const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 3); + const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4); + const auto count = + Binary(state, OpIAdd, state.uint_type, Unary(state, OpBitCount, state.uint_type, count_low), + Unary(state, OpBitCount, state.uint_type, count_high)); + const auto first = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first, + ConstantU32(state, ScopeSubgroup), ballot}); + const auto is_first = + Binary(state, OpIEqual, state.bool_type, EmitSubgroupLocalInvocationId(state), first); + const auto storage_bounds = gds ? EmitGdsElementInBounds(state, index) : 0u; + const auto m0_bounds = + gds ? Binary(state, OpINotEqual, state.bool_type, size, ConstantU32(state, 0)) + : Binary(state, OpULessThan, state.bool_type, + ConstantU32(state, ctx.Memory(inst).offset + 3u), size); + const auto condition = AndCondition( + state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds))); + const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {append ? OpAtomicIAdd : OpAtomicISub, state.uint_type, value, + gds ? EmitGdsElementPointer(state, index) : EmitLdsElementPointer(state, index), + ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), + ConstantU32(state, MemorySemanticsNone), count}); + return value; + }); + const auto result = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, result, + ConstantU32(state, ScopeSubgroup), atomic, first}); + return result; +} + +} // namespace + +bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto op = inst.GetOpcode(); + if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) && + ctx.Memory(inst).planning_only) { + return true; + } + if (op == IR::ValueOpcode::ReadConst) { + if (state.flattened_srt_variable == 0) { + ctx.Fail(inst, "requires the flattened SRT descriptor"); + return true; + } + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, + state.flattened_srt_variable, ConstantU32(state, 0), + ctx.Arg(inst, 1)}); + ctx.Emit(inst, OpLoad, IR::Type::U32, {pointer}); + return true; + } + if (op == IR::ValueOpcode::ReadConstBuffer) { + auto mem = ctx.Memory(inst); + mem.kind = IR::ResourceKind::ScalarBuffer; + const auto address = Binary(state, OpIAdd, state.uint_type, ctx.Arg(inst, 1), + ConstantU32(state, mem.offset)); + const auto index = + Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2)); + const auto pc = inst.Flags().pc; + const auto condition = EmitStorageBufferElementInBounds(state, mem, index, pc); + ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction( + {OpLoad, state.uint_type, value, + EmitStorageBufferElementPointer(state, mem, index, pc)}); + return value; + })); + return true; + } + const bool load_address = op == IR::ValueOpcode::LoadAddressU8 || + op == IR::ValueOpcode::LoadAddressU16 || + op == IR::ValueOpcode::LoadAddressU32; + const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 || + op == IR::ValueOpcode::LoadBufferU16 || + op == IR::ValueOpcode::LoadBufferU32; + const bool load_shared = + op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 || + op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 || + op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32; + if (load_address || load_buffer || load_shared) { + auto mem = ctx.Memory(inst); + if (load_shared) + mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 || + op == IR::ValueOpcode::LoadGdsU32 + ? IR::ResourceKind::Gds + : IR::ResourceKind::Lds; + uint32_t value = 0; + if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted) + value = FormattedLoad(ctx, inst, mem); + else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 || + op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8) + value = LoadSubword(ctx, inst, mem, 8, false); + else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 || + op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16) + value = LoadSubword(ctx, inst, mem, 16, false); + else + value = LoadWord(ctx, inst, mem); + ctx.Define(inst, value); + return true; + } + const bool store_address = op == IR::ValueOpcode::StoreAddressU8 || + op == IR::ValueOpcode::StoreAddressU16 || + op == IR::ValueOpcode::StoreAddressU32; + const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 || + op == IR::ValueOpcode::StoreBufferU16 || + op == IR::ValueOpcode::StoreBufferU32; + const bool store_shared = + op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 || + op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 || + op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32; + if (store_address || store_buffer || store_shared) { + auto mem = ctx.Memory(inst); + if (store_shared) + mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 || + op == IR::ValueOpcode::WriteGdsU32 + ? IR::ResourceKind::Gds + : IR::ResourceKind::Lds; + if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted) + FormattedStore(ctx, inst, mem); + else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 || + op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8) + StoreSubword(ctx, inst, mem, 8); + else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 || + op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16) + StoreSubword(ctx, inst, mem, 16); + else + StoreWord(ctx, inst, mem); + return true; + } + const auto atomic_opcode = AtomicOpcode(op); + if (atomic_opcode != 0) { + auto mem = ctx.Memory(inst); + if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32) + mem.kind = IR::ResourceKind::Lds; + else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32) + mem.kind = IR::ResourceKind::Gds; + ctx.Define(inst, EmitAtomic(ctx, inst, mem, true)); + return true; + } + if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) { + ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst), + op == IR::ValueOpcode::BufferAtomicFMax32)); + return true; + } + if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 || + op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) { + auto mem = ctx.Memory(inst); + mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32 + ? IR::ResourceKind::Gds + : IR::ResourceKind::Lds; + SharedFloatAtomic(ctx, inst, mem, + op == IR::ValueOpcode::SharedAtomicFMax32 || + op == IR::ValueOpcode::GdsAtomicFMax32); + return true; + } + if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume || + op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) { + ctx.Define(inst, AppendConsume(ctx, inst, + op == IR::ValueOpcode::GdsDataAppend || + op == IR::ValueOpcode::GdsDataConsume, + op == IR::ValueOpcode::DataAppend || + op == IR::ValueOpcode::GdsDataAppend)); + return true; + } + if (op == IR::ValueOpcode::SwizzleU32) { + ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)}); + const auto source = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, ctx.state.uint_type, source, ctx.scratch_u32_variable}); + const auto target = + EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state), + inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0); + const auto shuffled = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled, + ConstantU32(state, ScopeSubgroup), source, target}); + const auto source_exec = state.builder.AllocateId(); + state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec, + ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 2), target}); + const auto source_active = + AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target)); + ctx.Define(inst, + Select(state, state.uint_type, source_active, shuffled, ConstantU32(state, 0))); + return true; + } + return false; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterValues.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp similarity index 99% rename from src/graphics/shader/recompiler/emitter/spirvEmitterValues.cpp rename to src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp index bed80d628..f62731ace 100644 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterValues.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterValues.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { @@ -190,7 +190,8 @@ uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) { uint32_t value = 0; switch (operand.kind) { case IR::OperandKind::ImmediateU32: - case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break; + case IR::OperandKind::PcRelativeU32: + case IR::OperandKind::PcRelativeHighU32: value = ConstantU32(state, operand.imm); break; case IR::OperandKind::Register: { const auto pointer = PointerForRegister(state, operand.reg); if (pointer == 0) { diff --git a/src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp b/src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp deleted file mode 100644 index e94e1c9a2..000000000 --- a/src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp +++ /dev/null @@ -1,525 +0,0 @@ -#include "graphics/shader/recompiler/emitter/SpirvEmitter.h" - -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" -#include "graphics/shader/recompiler/ir/SrtWalker.h" - -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::Spirv { - -namespace { - -bool Fail(std::string* error, const std::string& message) { - if (error != nullptr) { - *error = message; - } - return false; -} - -bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) { - using Kind = IR::DescriptorBindingKind; - using Dim = Decoder::ImageDimension; - if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) { - const bool integer = image.kind == IR::ResourceKind::ImageUint; - switch (image.dimension) { - case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true; - case Dim::Dim1DArray: - kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray; - return true; - case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true; - case Dim::Dim2DMsaa: - kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa; - return true; - case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true; - case Dim::Dim2DArray: - kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray; - return true; - case Dim::Dim2DMsaaArray: - kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray; - return true; - case Dim::Unknown: return false; - } - } - const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint; - if (image.kind != IR::ResourceKind::StorageImage && !uint_image) { - return false; - } - switch (image.dimension) { - case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true; - case Dim::Dim1DArray: - kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray; - return true; - case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true; - case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true; - case Dim::Dim2DArray: - kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray; - return true; - case Dim::Dim2DMsaa: - case Dim::Dim2DMsaaArray: return false; - case Dim::Unknown: return false; - } - return false; -} - -bool NeedsSampler(IR::Opcode op) { - return op == IR::Opcode::ImageSample || op == IR::Opcode::ImageGather4 || - op == IR::Opcode::ImageGetLod; -} - -bool IsBufferLoad(IR::Opcode op) { - switch (op) { - case IR::Opcode::BufferLoadUbyte: - case IR::Opcode::BufferLoadSbyte: - case IR::Opcode::BufferLoadUshort: - case IR::Opcode::BufferLoadSshort: - case IR::Opcode::BufferLoadDword: return true; - default: return false; - } -} - -bool IsBufferStore(IR::Opcode op) { - return op == IR::Opcode::BufferStoreByte || op == IR::Opcode::BufferStoreShort || - op == IR::Opcode::BufferStoreDword; -} - -bool IsFlatLoad(IR::Opcode op) { - return op == IR::Opcode::FlatLoadUbyte || op == IR::Opcode::FlatLoadSbyte || - op == IR::Opcode::FlatLoadUshort || op == IR::Opcode::FlatLoadSshort || - op == IR::Opcode::FlatLoadDword; -} - -bool IsFlatStore(IR::Opcode op) { - return op == IR::Opcode::FlatStoreByte || op == IR::Opcode::FlatStoreShort || - op == IR::Opcode::FlatStoreDword; -} - -bool IsDsKind(IR::ResourceKind kind) { - return kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds; -} - -bool IsDsRead(IR::Opcode op) { - switch (op) { - case IR::Opcode::DsReadUbyte: - case IR::Opcode::DsReadSbyte: - case IR::Opcode::DsReadUshort: - case IR::Opcode::DsReadSshort: - case IR::Opcode::DsReadB32: return true; - default: return false; - } -} - -bool IsDsWrite(IR::Opcode op) { - switch (op) { - case IR::Opcode::DsWriteByte: - case IR::Opcode::DsWriteShort: - case IR::Opcode::DsWriteB32: return true; - default: return false; - } -} - -bool IsAtomic(IR::Opcode op) { - switch (op) { - case IR::Opcode::AtomicSwapU32: - case IR::Opcode::AtomicAddU32: - case IR::Opcode::AtomicSubU32: - case IR::Opcode::AtomicSMinI32: - case IR::Opcode::AtomicUMinU32: - case IR::Opcode::AtomicSMaxI32: - case IR::Opcode::AtomicUMaxU32: - case IR::Opcode::AtomicAndU32: - case IR::Opcode::AtomicOrU32: - case IR::Opcode::AtomicXorU32: - case IR::Opcode::AtomicFMinF32: - case IR::Opcode::AtomicFMaxF32: return true; - default: return false; - } -} - -uint32_t BufferAddressOperandCount(const IR::Instruction& inst) { - return 1u + (inst.memory.idxen ? 1u : 0u) + (inst.memory.offen ? 1u : 0u); -} - -bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error) { - if (static_cast(inst.op) >= static_cast(IR::Opcode::Count)) { - return Fail(error, "IR instruction opcode is out of range"); - } - if (inst.src_count > std::size(inst.src)) { - return Fail(error, "instruction source count exceeds fixed IR storage"); - } - const auto kind = inst.memory.kind; - const auto flat_kind = kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global || - kind == IR::ResourceKind::Scratch; - const auto buffer_address_count = BufferAddressOperandCount(inst); - if ((IsBufferLoad(inst.op) && - (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count)) || - (IsBufferStore(inst.op) && - (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) || - (inst.op == IR::Opcode::SLoadDword && - (kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) || - (inst.op == IR::Opcode::SBufferLoadDword && - (kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) || - (IsFlatLoad(inst.op) && (!flat_kind || inst.src_count != 2)) || - (IsFlatStore(inst.op) && (!flat_kind || inst.src_count != 3))) { - return Fail(error, "memory instruction opcode/kind/operand contract is invalid"); - } - const auto ds_kind = IsDsKind(kind); - const auto ds_resource = !ds_kind || inst.memory.resource == 0; - if ((IsDsRead(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 1 || - inst.dst.kind != IR::OperandKind::Register)) || - (IsDsWrite(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 2 || - inst.dst.kind != IR::OperandKind::Null)) || - ((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) && - (!ds_kind || !ds_resource || inst.src_count != 1 || - inst.dst.kind != IR::OperandKind::Register)) || - ((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) && - (!ds_kind || !ds_resource || inst.src_count != 3 || - inst.dst.kind != IR::OperandKind::Null)) || - (inst.op == IR::Opcode::DsWriteAddtidB32 && - (kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 2 || - inst.dst.kind != IR::OperandKind::Null)) || - (inst.op == IR::Opcode::DsReadAddtidB32 && - (kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 1 || - inst.dst.kind != IR::OperandKind::Register)) || - (inst.op == IR::Opcode::DsSwizzleB32 && - (kind != IR::ResourceKind::None || inst.src_count != 2 || - inst.dst.kind != IR::OperandKind::Register))) { - return Fail(error, "DS instruction has an invalid resource/operand contract"); - } - if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4 || - inst.op == IR::Opcode::ImageGetLod || inst.op == IR::Opcode::ImageLoad || - inst.op == IR::Opcode::ImageGetResinfo) && - ((kind != IR::ResourceKind::Image && kind != IR::ResourceKind::ImageUint) || - inst.src_count != 1)) { - return Fail(error, "sampled image instruction has an invalid resource class"); - } - if (inst.op == IR::Opcode::ImageGather4 && - (inst.memory.image_dimension == Decoder::ImageDimension::Dim1D || - inst.memory.image_dimension == Decoder::ImageDimension::Dim1DArray)) { - return Fail(error, "1D image gather is not supported by SPIR-V"); - } - if (inst.op == IR::Opcode::ImageStore && - ((kind != IR::ResourceKind::StorageImage && kind != IR::ResourceKind::StorageImageUint) || - inst.src_count != 2)) { - return Fail(error, "storage image instruction has an invalid resource class"); - } - const auto float_atomic = - inst.op == IR::Opcode::AtomicFMinF32 || inst.op == IR::Opcode::AtomicFMaxF32; - if (float_atomic && - (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) { - return Fail(error, "floating-point atomic instruction must target a buffer"); - } - if (IsAtomic(inst.op) && - !((kind == IR::ResourceKind::Buffer && inst.src_count == buffer_address_count + 1u) || - ((kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds) && - inst.memory.resource == 0 && inst.src_count == 2 && - (inst.dst.kind == IR::OperandKind::Null || - inst.dst.kind == IR::OperandKind::Register)) || - (kind == IR::ResourceKind::StorageImageUint && inst.src_count == 2))) { - return Fail(error, "atomic instruction has an invalid resource/operand contract"); - } - return true; -} - -bool ValidateNativeProgram(const IR::Program& program, std::string* error) { - using Kind = IR::DescriptorBindingKind; - constexpr auto KindCount = static_cast(Kind::Count); - std::array, KindCount> expected; - std::array present {}; - const auto Dense = [](size_t size) { - std::vector values(size); - for (uint32_t i = 0; i < values.size(); i++) { - values[i] = i; - } - return values; - }; - auto Expect = [&](Kind kind, std::vector resources = {}) { - const auto index = static_cast(kind); - present[index] = true; - expected[index] = std::move(resources); - }; - if (!program.info.buffers.empty()) { - Expect(Kind::Buffers, Dense(program.info.buffers.size())); - } - for (uint32_t i = 0; i < program.info.images.size(); i++) { - Kind kind; - if (!ImageBinding(program.info.images[i], kind)) { - return Fail(error, "native shader plan has an invalid image class"); - } - present[static_cast(kind)] = true; - expected[static_cast(kind)].push_back(i); - } - if (!program.info.samplers.empty()) { - Expect(Kind::Samplers, Dense(program.info.samplers.size())); - } - const auto uses_gds = - std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) { - return std::any_of( - block.instructions.begin(), block.instructions.end(), - [](const auto& inst) { return inst.memory.kind == IR::ResourceKind::Gds; }); - }); - if (uses_gds) { - Expect(Kind::Gds); - } - if (!program.info.addresses.empty()) { - Expect(Kind::AddressMemory, Dense(program.info.addresses.size())); - } - if (!program.srt.reads.empty()) { - Expect(Kind::FlattenedSrt); - } - if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) { - Expect(Kind::UserData); - } - - std::array seen {}; - for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) { - const auto& binding = program.bindings.descriptors[i]; - const auto kind = static_cast(binding.kind); - if (kind >= KindCount || seen[kind] || !present[kind] || - binding.resources != expected[kind]) { - return Fail(error, "native descriptor groups do not match shader topology"); - } - for (uint32_t j = 0; j < i; j++) { - if (program.bindings.descriptors[j].binding == binding.binding) { - return Fail(error, "native descriptor binding numbers are not unique"); - } - } - seen[kind] = true; - } - for (size_t i = 0; i < KindCount; i++) { - if (present[i] != seen[i]) { - return Fail(error, "native shader plan is missing a required descriptor group"); - } - } - const auto has_user_storage = present[static_cast(Kind::UserData)]; - if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() || - program.bindings.buffer_offset_count != program.info.buffers.size() || - (!has_user_storage && - program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) || - (has_user_storage && program.bindings.push_constant_size != 0) || - !std::is_sorted(program.bindings.user_data_registers.begin(), - program.bindings.user_data_registers.end()) || - std::adjacent_find(program.bindings.user_data_registers.begin(), - program.bindings.user_data_registers.end()) != - program.bindings.user_data_registers.end()) { - return Fail(error, "native user-data layout is inconsistent"); - } - - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (!ValidateInstructionContract(inst, error)) { - return false; - } - const auto memory = inst.memory.kind; - if ((memory == IR::ResourceKind::Buffer || (memory == IR::ResourceKind::ScalarBuffer && - inst.op == IR::Opcode::SBufferLoadDword)) && - inst.memory.resource >= program.info.buffers.size()) { - return Fail(error, "buffer instruction has an invalid dense resource"); - } - if ((memory == IR::ResourceKind::Image || memory == IR::ResourceKind::ImageUint || - memory == IR::ResourceKind::StorageImage || - memory == IR::ResourceKind::StorageImageUint) && - (inst.memory.resource >= program.info.images.size() || - program.info.images[inst.memory.resource].kind != memory || - program.info.images[inst.memory.resource].dimension != - inst.memory.image_dimension)) { - return Fail(error, "image instruction has an invalid dense resource"); - } - const bool address = - inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat || - memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch; - if (address && (inst.memory.resource >= program.info.addresses.size() || - program.info.addresses[inst.memory.resource].kind != memory)) { - return Fail(error, "address instruction has an invalid dense resource"); - } - if (NeedsSampler(inst.op) && inst.memory.sampler >= program.info.samplers.size()) { - return Fail(error, "sample instruction has an invalid dense sampler"); - } - if (inst.op == IR::Opcode::LoadSrtDword && - (inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32 || - inst.src[0].imm >= program.srt.reads.size())) { - return Fail(error, "flattened SRT instruction has an invalid dense slot"); - } - } - } - return true; -} - -} // namespace - -static constexpr size_t InitialEmitterVectorReserve = 4096; - -static void ReportReserveExceeded(const IR::Program& program, const char* vector_name, - size_t size) { - if (size <= InitialEmitterVectorReserve) { - return; - } - std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64 - " stage=%u vector=%s size=%zu reserve=%zu\n", - program.shader_hash, static_cast(program.stage), vector_name, size, - InitialEmitterVectorReserve); -} - -static bool IsExecDestination(const IR::Operand& operand) { - return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Exec; -} - -static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruction& inst) { - if (inst.op != IR::Opcode::BitFieldMaskU64 || !IsExecDestination(inst.dst) || - inst.src_count < 1u) { - return false; - } - uint32_t count = 0; - return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], count) && - (count & 63u) == 0u; -} - -static bool ProducesLaneMask(const IR::Instruction& inst) { - switch (inst.op) { - case IR::Opcode::IAddCarryU32: - case IR::Opcode::ISubBorrowU32: - case IR::Opcode::UMadU64U32: return inst.dst2.kind != IR::OperandKind::Null; - default: return false; - } -} - -bool ProgramRequiresExactSubgroupSize(const IR::Program& program) { - for (const auto& block: program.blocks) { - if (block.terminator.kind == CFG::TerminatorKind::ConditionalBranch) { - switch (block.terminator.condition) { - case CFG::BranchCondition::VccZero: - case CFG::BranchCondition::VccNonZero: - case CFG::BranchCondition::ExecZero: - case CFG::BranchCondition::ExecNonZero: return true; - default: break; - } - } - for (const auto& inst: block.instructions) { - if (Emitter::InstructionHasDppSource(inst)) { - return true; - } - if (IR::IsCompareOpcode(inst.op) && !Emitter::IsSccOperand(inst.dst)) { - return true; - } - if ((IsExecDestination(inst.dst) || IsExecDestination(inst.dst2)) && - !IsUniformZeroExecWrite(program, inst)) { - return true; - } - if (ProducesLaneMask(inst)) { - return true; - } - if (inst.op == IR::Opcode::SelectMaskU32 || inst.op == IR::Opcode::SelectMaskF32Bits) { - return true; - } - if (inst.op == IR::Opcode::SaveexecB32 || inst.op == IR::Opcode::SaveexecB64) { - return true; - } - switch (inst.op) { - case IR::Opcode::WqmB64: - if (inst.dst.kind == IR::OperandKind::Register && - inst.dst.reg.file != IR::RegisterFile::Scalar) { - return true; - } - break; - case IR::Opcode::MaskedBitCountLowU32: - case IR::Opcode::MaskedBitCountHighU32: - case IR::Opcode::ReadFirstLaneU32: - case IR::Opcode::ReadLaneU32: - case IR::Opcode::WriteLaneU32: - case IR::Opcode::Permlane16B32: - case IR::Opcode::Permlanex16B32: - case IR::Opcode::DsSwizzleB32: - case IR::Opcode::DsConsume: - case IR::Opcode::DsAppend: return true; - default: break; - } - } - } - return false; -} - -bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, - const ShaderVertexInputInfo* vertex_input_info, - const ShaderPixelInputInfo* pixel_input_info, - const ShaderComputeInputInfo* compute_input_info, std::vector& spirv, - std::string* error) { - using namespace Emitter; - - if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && - program.stage != ShaderType::Pixel) { - SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders"); - return false; - } - if (!program.srt_patching_complete || !program.resource_tracking_complete || - !program.shader_info_complete || !program.binding_layout_complete) { - SetError(error, "SPIR-V emitter requires a fully planned native shader program"); - return false; - } - if (!IR::ValidateResourceSnapshot(program, resources, error)) { - return false; - } - if (!IR::ValidateResourceSpecialization(program, resources, error)) { - return false; - } - if (!ValidateNativeProgram(program, error)) { - return false; - } - - EmitterState state(program, resources); - state.vertex_input_info = vertex_input_info; - state.pixel_input_info = pixel_input_info; - state.compute_input_info = compute_input_info; - state.stage = program.stage; - state.wave_size = program.wave_size; - state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; - state.exact_subgroup_operations = - !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program); - state.registers.reserve(InitialEmitterVectorReserve); - state.inputs.reserve(InitialEmitterVectorReserve); - state.outputs.reserve(InitialEmitterVectorReserve); - state.interface_variables.reserve(InitialEmitterVectorReserve); - state.reachable_blocks.reserve(InitialEmitterVectorReserve); - CollectRegisters(program, state.registers); - CopyProgramInputsAndOutputs(state, program); - state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program); - state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program); - state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program); - state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program); - state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program); - state.needs_function_lds = ProgramNeedsFunctionLds(program); - state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program); - ComputeReachableBlocks(state, program); - AllocateInputVariables(state); - AllocateOutputVariables(state); - AllocateDescriptorVariables(state); - EmitHeaderAndTypes(state); - AllocateRegisterVariables(state); - AllocateBlockLabels(state, program); - AllocateDispatcherState(state, program); - EmitFunction(state, program); - if (state.unsupported_ir_instruction) { - char pc[9] {}; - std::snprintf(pc, sizeof(pc), "%08x", state.unsupported_ir_pc); - return Fail(error, "SPIR-V emitter has no handler for IR opcode " + - std::string(IR::OpcodeName(state.unsupported_ir_opcode)) + - " at pc 0x" + pc); - } - - ReportReserveExceeded(program, "registers", state.registers.size()); - ReportReserveExceeded(program, "inputs", state.inputs.size()); - ReportReserveExceeded(program, "outputs", state.outputs.size()); - ReportReserveExceeded(program, "interface_variables", state.interface_variables.size()); - ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size()); - - auto binary = state.builder.Build(); - if (binary.empty()) { - SetError(error, "SPIR-V builder returned an empty module"); - return false; - } - spirv = std::move(binary); - return true; -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp b/src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp deleted file mode 100644 index 58df9b67d..000000000 --- a/src/graphics/shader/recompiler/emitter/spirvEmitterControlFlow.cpp +++ /dev/null @@ -1,1138 +0,0 @@ -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" - -namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { - -void ComputeReachableBlocks(EmitterState& state, const IR::Program& program) { - state.reachable_blocks.assign(program.blocks.size(), false); - if (program.blocks.empty()) { - return; - } - - std::vector stack; - stack.push_back(0); - while (!stack.empty()) { - const auto block_id = stack.back(); - stack.pop_back(); - if (block_id >= program.blocks.size() || state.reachable_blocks[block_id]) { - continue; - } - state.reachable_blocks[block_id] = true; - for (auto succ: program.blocks[block_id].successors) { - if (succ < program.blocks.size() && !state.reachable_blocks[succ]) { - stack.push_back(succ); - } - } - } -} - -void AllocateBlockLabels(EmitterState& state, const IR::Program& program) { - for (const auto& block: program.blocks) { - if (block.id < state.reachable_blocks.size() && state.reachable_blocks[block.id]) { - state.block_labels.emplace(block.id, state.builder.AllocateId()); - } - } -} - -void AllocateDispatcherState(EmitterState& state, const IR::Program& program) { - state.dispatcher_fallback = program.dispatcher_fallback; - if (!state.dispatcher_fallback) { - return; - } - state.dispatch_pc_variable = state.builder.AllocateId(); - state.dispatch_header_label = state.builder.AllocateId(); - state.dispatch_select_label = state.builder.AllocateId(); - state.dispatch_default_label = state.builder.AllocateId(); - state.dispatch_after_switch_label = state.builder.AllocateId(); - state.dispatch_continue_label = state.builder.AllocateId(); - state.dispatch_merge_label = state.builder.AllocateId(); -} - -uint32_t BlockLabel(const EmitterState& state, uint32_t block_id) { - auto it = state.block_labels.find(block_id); - return it == state.block_labels.end() ? 0 : it->second; -} - -uint32_t EmitBranchCondition(EmitterState& state, CFG::BranchCondition condition) { - switch (condition) { - case CFG::BranchCondition::SccZero: return EmitSccBool(state, false); - case CFG::BranchCondition::SccNonZero: return EmitSccBool(state, true); - case CFG::BranchCondition::VccZero: - return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Vcc, true); - case CFG::BranchCondition::VccNonZero: - return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Vcc, false); - case CFG::BranchCondition::ExecZero: - return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Exec, true); - case CFG::BranchCondition::ExecNonZero: - return EmitMaskSummaryZeroBool(state, IR::RegisterFile::Exec, false); - default: return EmitSccBool(state, true); - } -} - -void EmitStructuredPrefix(EmitterState& state, const CFG::Terminator& term) { - if (term.loop_header) { - const auto merge_label = BlockLabel(state, term.merge_block); - const auto continue_label = BlockLabel(state, term.continue_block); - if (merge_label != 0 && continue_label != 0) { - state.builder.AddFunction({OpLoopMerge, merge_label, continue_label, LoopControlNone}); - } - return; - } - if (term.kind == CFG::TerminatorKind::ConditionalBranch && term.merge_block != UINT32_MAX) { - const auto merge_label = BlockLabel(state, term.merge_block); - if (merge_label != 0) { - state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); - } - } -} - -void EmitReturn(EmitterState& state) { - EmitKillIfPixelValidMaskInactive(state); - state.builder.AddFunction({OpReturn}); -} - -void EmitTerminator(EmitterState& state, const CFG::Terminator& term) { - const auto condition = term.kind == CFG::TerminatorKind::ConditionalBranch - ? EmitBranchCondition(state, term.condition) - : 0; - EmitStructuredPrefix(state, term); - - switch (term.kind) { - case CFG::TerminatorKind::Branch: { - const auto label = BlockLabel(state, term.true_block); - if (label != 0) { - state.builder.AddFunction({OpBranch, label}); - } else { - EmitReturn(state); - } - break; - } - case CFG::TerminatorKind::ConditionalBranch: { - const auto true_label = BlockLabel(state, term.true_block); - const auto false_label = BlockLabel(state, term.false_block); - if (true_label != 0 && false_label != 0) { - state.builder.AddFunction( - {OpBranchConditional, condition, true_label, false_label}); - } else { - EmitReturn(state); - } - break; - } - default: EmitReturn(state); break; - } -} - -bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) { - if (reg.file != IR::RegisterFile::Scalar) { - return false; - } - const auto& registers = state.program.bindings.user_data_registers; - const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index); - if (found != registers.end() && *found == reg.index) { - dword_index = static_cast(found - registers.begin()); - return true; - } - return false; -} - -uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { - if (state.push_constant_variable != 0) { - const auto pointer = state.builder.AllocateId(); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer, - state.push_constant_variable, ConstantU32(state, 0), - ConstantU32(state, dword_index)}); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - return value; - } - if (state.vsharp_storage_variable != 0) { - const auto pointer = state.builder.AllocateId(); - const auto value = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer, - state.vsharp_storage_variable, ConstantU32(state, 0), - ConstantU32(state, dword_index)}); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - return value; - } - return ConstantU32(state, 0); -} - -uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) { - if (reg.file == IR::RegisterFile::Exec) { - if (state.per_invocation_masks) { - return reg.index == 0 ? 1u : 0u; - } - if (reg.index == 0) { - return 0xffffffffu; - } - return state.wave_size == 64u ? 0xffffffffu : 0u; - } - return 0; -} - -uint32_t EmitInitialRegisterValue(EmitterState& state, IR::Register reg) { - uint32_t dword_index = 0; - if (UserDataDwordIndex(state, reg, dword_index)) { - return EmitShaderDataDwordLoad(state, dword_index); - } - return ConstantU32(state, InitialRegisterValue(state, reg)); -} - -void EmitRegisterVariables(EmitterState& state) { - if (state.needs_function_lds) { - state.builder.AddFunction( - {OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction}); - } - for (const auto& binding: state.registers) { - state.builder.AddFunction( - {OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction}); - } - if (state.dispatcher_fallback) { - state.builder.AddFunction( - {OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction}); - } - if (state.pixel_valid_mask_variable != 0) { - state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable, - StorageClassFunction}); - state.builder.AddFunction( - {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)}); - } - for (const auto& binding: state.registers) { - state.builder.AddFunction( - {OpStore, binding.pointer_id, EmitInitialRegisterValue(state, binding.reg)}); - } -} - -void EmitComputeInputRegisters(EmitterState& state) { - if (state.stage != ShaderType::Compute || state.compute_input_info == nullptr) { - return; - } - - const auto* cs = state.compute_input_info; - const auto thread_ids = cs->thread_ids_num > 0 - ? std::min(static_cast(cs->thread_ids_num), 3u) - : 0u; - for (uint32_t i = 0; i < thread_ids; i++) { - const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, i}); - if (pointer != 0) { - state.builder.AddFunction( - {OpStore, pointer, - EmitInputComponentU32(state, IR::StageInputKind::LocalInvocationId, i)}); - } - } - - uint32_t reg_offset = 0; - for (uint32_t i = 0; i < 3u; i++) { - if (!cs->group_id[i]) { - continue; - } - const auto pointer = - PointerForRegister(state, {IR::RegisterFile::Scalar, - static_cast(cs->workgroup_register) + reg_offset}); - if (pointer != 0) { - state.builder.AddFunction( - {OpStore, pointer, - EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, i)}); - } - reg_offset++; - } - - if (cs->tg_size_en) { - const auto pointer = - PointerForRegister(state, {IR::RegisterFile::Scalar, - static_cast(cs->workgroup_register) + reg_offset}); - if (pointer != 0) { - const uint32_t wave_size = cs->wave_size != 0 ? cs->wave_size : 64u; - const uint32_t total_threads = std::max(cs->threads_num[0], 1u) * - std::max(cs->threads_num[1], 1u) * - std::max(cs->threads_num[2], 1u); - const uint32_t waves = - std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu); - - const auto local_index = EmitLocalInvocationIndex(state); - const auto wave_id = state.builder.AllocateId(); - const auto wave_bits = state.builder.AllocateId(); - const auto is_first = state.builder.AllocateId(); - const auto first_bit = state.builder.AllocateId(); - const auto base = state.builder.AllocateId(); - const auto packed = state.builder.AllocateId(); - state.builder.AddFunction( - {OpUDiv, state.uint_type, wave_id, local_index, ConstantU32(state, wave_size)}); - state.builder.AddFunction( - {OpShiftLeftLogical, state.uint_type, wave_bits, wave_id, ConstantU32(state, 20)}); - state.builder.AddFunction( - {OpIEqual, state.bool_type, is_first, wave_id, ConstantU32(state, 0)}); - state.builder.AddFunction({OpSelect, state.uint_type, first_bit, is_first, - ConstantU32(state, 0x80000000u), ConstantU32(state, 0)}); - state.builder.AddFunction( - {OpBitwiseOr, state.uint_type, base, wave_bits, ConstantU32(state, waves)}); - state.builder.AddFunction({OpBitwiseOr, state.uint_type, packed, base, first_bit}); - state.builder.AddFunction({OpStore, pointer, packed}); - } - } -} - -void EmitPixelFragCoordComponent(EmitterState& state, uint32_t component, uint32_t vgpr) { - const auto input = InputVariableForKind(state, IR::StageInputKind::FragCoord); - const auto dst = PointerForRegister(state, {IR::RegisterFile::Vector, vgpr}); - if (input == 0 || dst == 0) { - return; - } - - const auto pointer = state.builder.AllocateId(); - const auto value = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction( - {OpAccessChain, state.ptr_input_float, pointer, input, ConstantU32(state, component)}); - state.builder.AddFunction({OpLoad, state.float_type, value, pointer}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, value}); - state.builder.AddFunction({OpStore, dst, bits}); -} - -void EmitPixelFrontFacingRegister(EmitterState& state, uint32_t vgpr) { - const auto input = InputVariableForKind(state, IR::StageInputKind::FrontFacing); - const auto dst = PointerForRegister(state, {IR::RegisterFile::Vector, vgpr}); - if (input == 0 || dst == 0) { - return; - } - - const auto value = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.bool_type, value, input}); - state.builder.AddFunction( - {OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)}); - state.builder.AddFunction({OpStore, dst, bits}); -} - -void EmitPixelInputRegisters(EmitterState& state) { - if (state.stage != ShaderType::Pixel || state.pixel_input_info == nullptr) { - return; - } - - const auto* ps = state.pixel_input_info; - uint32_t reg = ps->ps_system_input_base; - if (ps->HasPositionInput()) { - if (ps->ps_pos_x) { - EmitPixelFragCoordComponent(state, 0, reg); - reg++; - } - if (ps->ps_pos_y) { - EmitPixelFragCoordComponent(state, 1, reg); - reg++; - } - if (ps->ps_pos_z) { - EmitPixelFragCoordComponent(state, 2, reg); - reg++; - } - if (ps->ps_pos_w) { - EmitPixelFragCoordComponent(state, 3, reg); - reg++; - } - } - if (ps->ps_front_face) { - EmitPixelFrontFacingRegister(state, reg); - } -} - -uint32_t EmitInputScalarU32(EmitterState& state, IR::StageInputKind kind) { - const auto variable = InputVariableForKind(state, kind); - if (variable == 0) { - return ConstantU32(state, 0); - } - const auto raw = state.builder.AllocateId(); - const auto bits = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.int_type, raw, variable}); - state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw}); - return bits; -} - -void EmitVertexInputRegisters(EmitterState& state) { - if (state.stage != ShaderType::Vertex) { - return; - } - - const auto vertex_index = PointerForRegister(state, {IR::RegisterFile::Vector, 5}); - if (vertex_index != 0) { - state.builder.AddFunction( - {OpStore, vertex_index, EmitInputScalarU32(state, IR::StageInputKind::VertexIndex)}); - } - - const auto instance_index = PointerForRegister(state, {IR::RegisterFile::Vector, 8}); - if (instance_index != 0) { - state.builder.AddFunction({OpStore, instance_index, - EmitInputScalarU32(state, IR::StageInputKind::InstanceIndex)}); - } -} - -void EmitInstruction(EmitterState& state, const IR::Instruction& inst) { - switch (inst.op) { - case IR::Opcode::ControlNop: EmitControlNop(state, inst); break; - case IR::Opcode::Waitcnt: EmitWaitcnt(state, inst); break; - case IR::Opcode::Barrier: EmitBarrier(state, inst); break; - case IR::Opcode::Sendmsg: - case IR::Opcode::TtraceData: - case IR::Opcode::InstPrefetch: EmitControlNop(state, inst); break; - case IR::Opcode::LoadInputF32: EmitLoadInputF32(state, inst); break; - case IR::Opcode::MoveU32: EmitMoveU32(state, inst); break; - case IR::Opcode::MoveF32Bits: EmitMoveF32Bits(state, inst); break; - case IR::Opcode::MoveRelDestU32: EmitMoveRelDestU32(state, inst); break; - case IR::Opcode::MoveRelSourceU32: EmitMoveRelSourceU32(state, inst); break; - case IR::Opcode::MoveU64: EmitMoveU64(state, inst); break; - case IR::Opcode::WqmB64: EmitWqmB64(state, inst); break; - case IR::Opcode::SaveexecB32: EmitSaveexecB32(state, inst); break; - case IR::Opcode::SaveexecB64: EmitSaveexecB64(state, inst); break; - case IR::Opcode::ReadFirstLaneU32: EmitReadFirstLaneU32(state, inst); break; - case IR::Opcode::ReadLaneU32: EmitReadLaneU32(state, inst); break; - case IR::Opcode::WriteLaneU32: EmitWriteLaneU32(state, inst); break; - case IR::Opcode::Permlane16B32: EmitPermlaneB32(state, inst, false); break; - case IR::Opcode::Permlanex16B32: EmitPermlaneB32(state, inst, true); break; - case IR::Opcode::AbsI32: EmitAbsI32(state, inst); break; - case IR::Opcode::IAddU32: EmitBinaryU32(state, inst, OpIAdd); break; - case IR::Opcode::IAddCarryU32: EmitIAddCarryU32(state, inst); break; - case IR::Opcode::ISubBorrowU32: EmitISubBorrowU32(state, inst); break; - case IR::Opcode::ScalarAddCarryU32: EmitScalarAddCarryU32(state, inst); break; - case IR::Opcode::ScalarSubBorrowU32: EmitScalarSubBorrowU32(state, inst); break; - case IR::Opcode::ScalarSubBorrowCarryU32: EmitScalarSubBorrowCarryU32(state, inst); break; - case IR::Opcode::ScalarSignedAddOverflowI32: - EmitScalarSignedOverflowI32(state, inst, false); - break; - case IR::Opcode::ScalarSignedSubOverflowI32: - EmitScalarSignedOverflowI32(state, inst, true); - break; - case IR::Opcode::ScalarShiftLeftAddCarryU32: - EmitScalarShiftLeftAddCarryU32(state, inst); - break; - case IR::Opcode::ISubU32: EmitBinaryU32(state, inst, OpISub); break; - case IR::Opcode::IMulU32: EmitBinaryU32(state, inst, OpIMul); break; - case IR::Opcode::UMulHighU32: EmitMulHighU32(state, inst); break; - case IR::Opcode::SMulHighI32: EmitMulHighI32(state, inst); break; - case IR::Opcode::IMadI24U32: EmitMadI32I24(state, inst); break; - case IR::Opcode::UMadU24U32: EmitMadU32U24(state, inst); break; - case IR::Opcode::UMadU64U32: EmitUMadU64U32(state, inst); break; - case IR::Opcode::SadU32: EmitSadU32(state, inst); break; - case IR::Opcode::IAdd3U32: EmitAdd3U32(state, inst); break; - case IR::Opcode::IMulI24U32: EmitMulI24U32(state, inst); break; - case IR::Opcode::UMulU24U32: EmitMulU24U32(state, inst); break; - case IR::Opcode::IMinI32: EmitMinMaxI32(state, inst, false); break; - case IR::Opcode::IMaxI32: EmitMinMaxI32(state, inst, true); break; - case IR::Opcode::IMin3I32: EmitMinMax3U32(state, inst, true, false); break; - case IR::Opcode::IMax3I32: EmitMinMax3U32(state, inst, true, true); break; - case IR::Opcode::IMed3I32: EmitMed3U32(state, inst, true); break; - case IR::Opcode::UMinU32: EmitMinMaxU32(state, inst, false); break; - case IR::Opcode::UMaxU32: EmitMinMaxU32(state, inst, true); break; - case IR::Opcode::UMin3U32: EmitMinMax3U32(state, inst, false, false); break; - case IR::Opcode::UMax3U32: EmitMinMax3U32(state, inst, false, true); break; - case IR::Opcode::UMed3U32: EmitMed3U32(state, inst, false); break; - case IR::Opcode::BitwiseAndU32: EmitBinaryU32(state, inst, OpBitwiseAnd); break; - case IR::Opcode::BitwiseAndU64: EmitBinaryU64(state, inst, OpBitwiseAnd); break; - case IR::Opcode::BitwiseAndNotU32: EmitBinaryNotRhsU32(state, inst, OpBitwiseAnd); break; - case IR::Opcode::BitwiseAndNotU64: EmitBinaryNotRhsU64(state, inst, OpBitwiseAnd); break; - case IR::Opcode::BitwiseOrU32: EmitBinaryU32(state, inst, OpBitwiseOr); break; - case IR::Opcode::BitwiseOrU64: EmitBinaryU64(state, inst, OpBitwiseOr); break; - case IR::Opcode::BitwiseOrNotU32: EmitBinaryNotRhsU32(state, inst, OpBitwiseOr); break; - case IR::Opcode::BitwiseOrNotU64: EmitBinaryNotRhsU64(state, inst, OpBitwiseOr); break; - case IR::Opcode::BitwiseAndOrU32: - EmitChainedBinaryU32(state, inst, OpBitwiseAnd, OpBitwiseOr); - break; - case IR::Opcode::BitwiseOr3U32: - EmitChainedBinaryU32(state, inst, OpBitwiseOr, OpBitwiseOr); - break; - case IR::Opcode::BitwiseXorU32: EmitBinaryU32(state, inst, OpBitwiseXor); break; - case IR::Opcode::BitwiseXorU64: EmitBinaryU64(state, inst, OpBitwiseXor); break; - case IR::Opcode::BitwiseXor3U32: - EmitChainedBinaryU32(state, inst, OpBitwiseXor, OpBitwiseXor); - break; - case IR::Opcode::BitwiseNandU32: EmitBinaryThenNotU32(state, inst, OpBitwiseAnd); break; - case IR::Opcode::BitwiseNandU64: EmitBinaryThenNotU64(state, inst, OpBitwiseAnd); break; - case IR::Opcode::BitwiseNorU32: EmitBinaryThenNotU32(state, inst, OpBitwiseOr); break; - case IR::Opcode::BitwiseNorU64: EmitBinaryThenNotU64(state, inst, OpBitwiseOr); break; - case IR::Opcode::BitwiseXnorU32: EmitBinaryThenNotU32(state, inst, OpBitwiseXor); break; - case IR::Opcode::BitwiseXnorU64: EmitBinaryThenNotU64(state, inst, OpBitwiseXor); break; - case IR::Opcode::BitwiseNotU32: EmitUnaryU32(state, inst, OpNot); break; - case IR::Opcode::BitwiseNotU64: EmitUnaryU64(state, inst, OpNot); break; - case IR::Opcode::BitClearU32: EmitBitClearU32(state, inst); break; - case IR::Opcode::BitSetU32: EmitBitSetU32(state, inst); break; - case IR::Opcode::BitReverseU32: EmitUnaryU32(state, inst, OpBitReverse); break; - case IR::Opcode::BitCountU32: EmitBitCountU32(state, inst); break; - case IR::Opcode::BitCountU64: EmitBitCountU64(state, inst); break; - case IR::Opcode::BitCountAddU32: EmitBitCountAddU32(state, inst); break; - case IR::Opcode::MaskedBitCountLowU32: EmitMaskedBitCountU32(state, inst, 0); break; - case IR::Opcode::MaskedBitCountHighU32: EmitMaskedBitCountU32(state, inst, 1); break; - case IR::Opcode::BitReplicateB64B32: EmitBitReplicateB64B32(state, inst); break; - case IR::Opcode::FindLsbU32: EmitFindLsbU32(state, inst); break; - case IR::Opcode::FindMsbFromHighU32: EmitFindMsbFromHighU32(state, inst); break; - case IR::Opcode::FindMsbFromHighU64: EmitFindMsbFromHighU64(state, inst); break; - case IR::Opcode::BitFieldMaskU32: EmitBitFieldMaskU32(state, inst); break; - case IR::Opcode::BitFieldMaskU64: EmitBitFieldMaskU64(state, inst); break; - case IR::Opcode::BitFieldExtractU32: EmitBitFieldExtractU32(state, inst); break; - case IR::Opcode::BitFieldExtractU64: EmitBitFieldExtractU64(state, inst); break; - case IR::Opcode::BitFieldExtract3U32: EmitBitFieldExtract3U32(state, inst, false); break; - case IR::Opcode::BitFieldExtract3I32: EmitBitFieldExtract3U32(state, inst, true); break; - case IR::Opcode::BitFieldInsertSelectU32: EmitBitFieldInsertSelectU32(state, inst); break; - case IR::Opcode::BitCompare0B32: EmitBitCompareB32(state, inst, false); break; - case IR::Opcode::BitCompare1B32: EmitBitCompareB32(state, inst, true); break; - case IR::Opcode::AlignBitU32: EmitAlignBitU32(state, inst); break; - case IR::Opcode::ShiftLeftAddU32: - EmitChainedBinaryU32(state, inst, OpShiftLeftLogical, OpIAdd, true); - break; - case IR::Opcode::AddShiftLeftU32: - EmitChainedBinaryU32(state, inst, OpIAdd, OpShiftLeftLogical, false, true); - break; - case IR::Opcode::XorAddU32: EmitChainedBinaryU32(state, inst, OpBitwiseXor, OpIAdd); break; - case IR::Opcode::ShiftLeftOrU32: - EmitChainedBinaryU32(state, inst, OpShiftLeftLogical, OpBitwiseOr, true); - break; - case IR::Opcode::ShiftLeftLogicalU32: EmitShiftU32(state, inst, OpShiftLeftLogical); break; - case IR::Opcode::ShiftLeftLogicalU64: EmitShiftLeftLogicalU64(state, inst); break; - case IR::Opcode::ShiftLeftLogicalU16: - EmitShiftU16(state, inst, OpShiftLeftLogical, false); - break; - case IR::Opcode::ShiftRightLogicalU32: - EmitShiftU32(state, inst, OpShiftRightLogical); - break; - case IR::Opcode::ShiftRightLogicalU64: EmitShiftRightLogicalU64(state, inst); break; - case IR::Opcode::ShiftRightLogicalU16: - EmitShiftU16(state, inst, OpShiftRightLogical, false); - break; - case IR::Opcode::ShiftRightArithmeticI32: - EmitShiftU32(state, inst, OpShiftRightArithmetic); - break; - case IR::Opcode::ShiftRightArithmeticI16: - EmitShiftU16(state, inst, OpShiftRightArithmetic, true); - break; - case IR::Opcode::SelectU32: EmitSelectU32(state, inst); break; - case IR::Opcode::SelectMaskU32: EmitSelectMaskU32(state, inst); break; - case IR::Opcode::SelectF32Bits: EmitSelectF32Bits(state, inst); break; - case IR::Opcode::SelectMaskF32Bits: EmitSelectMaskF32Bits(state, inst); break; - case IR::Opcode::SelectU64: EmitSelectU64(state, inst); break; - case IR::Opcode::PackLowLowU16: EmitPackU16(state, inst, false, false); break; - case IR::Opcode::PackLowHighU16: EmitPackU16(state, inst, false, true); break; - case IR::Opcode::PackHighHighU16: EmitPackU16(state, inst, true, true); break; - case IR::Opcode::PackU16U32: EmitPackU16(state, inst, false, false); break; - case IR::Opcode::CompareFalse: EmitCompareConstant(state, inst, false); break; - case IR::Opcode::CompareTrue: EmitCompareConstant(state, inst, true); break; - case IR::Opcode::CompareEqU32: EmitCompareU32(state, inst, OpIEqual); break; - case IR::Opcode::CompareNeU32: EmitCompareU32(state, inst, OpINotEqual); break; - case IR::Opcode::CompareGtU32: EmitCompareU32(state, inst, OpUGreaterThan); break; - case IR::Opcode::CompareGeU32: EmitCompareU32(state, inst, OpUGreaterThanEqual); break; - case IR::Opcode::CompareLtU32: EmitCompareU32(state, inst, OpULessThan); break; - case IR::Opcode::CompareLeU32: EmitCompareU32(state, inst, OpULessThanEqual); break; - case IR::Opcode::CompareEqU64: - case IR::Opcode::CompareNeU64: EmitCompareU64(state, inst); break; - case IR::Opcode::CompareMaskEqU32: EmitCompareMaskU32(state, inst, OpIEqual); break; - case IR::Opcode::CompareMaskNeU32: EmitCompareMaskU32(state, inst, OpINotEqual); break; - case IR::Opcode::CompareMaskGtU32: EmitCompareMaskU32(state, inst, OpUGreaterThan); break; - case IR::Opcode::CompareMaskGeU32: - EmitCompareMaskU32(state, inst, OpUGreaterThanEqual); - break; - case IR::Opcode::CompareMaskLtU32: EmitCompareMaskU32(state, inst, OpULessThan); break; - case IR::Opcode::CompareMaskLeU32: EmitCompareMaskU32(state, inst, OpULessThanEqual); break; - case IR::Opcode::CompareEqI32: EmitCompareU32(state, inst, OpIEqual); break; - case IR::Opcode::CompareNeI32: EmitCompareU32(state, inst, OpINotEqual); break; - case IR::Opcode::CompareGtI32: EmitCompareU32(state, inst, OpSGreaterThan); break; - case IR::Opcode::CompareGeI32: EmitCompareU32(state, inst, OpSGreaterThanEqual); break; - case IR::Opcode::CompareLtI32: EmitCompareU32(state, inst, OpSLessThan); break; - case IR::Opcode::CompareLeI32: EmitCompareU32(state, inst, OpSLessThanEqual); break; - case IR::Opcode::CompareEqI16: EmitCompareI16(state, inst, OpIEqual); break; - case IR::Opcode::CompareNeI16: EmitCompareI16(state, inst, OpINotEqual); break; - case IR::Opcode::CompareGtI16: EmitCompareI16(state, inst, OpSGreaterThan); break; - case IR::Opcode::CompareGeI16: EmitCompareI16(state, inst, OpSGreaterThanEqual); break; - case IR::Opcode::CompareLtI16: EmitCompareI16(state, inst, OpSLessThan); break; - case IR::Opcode::CompareLeI16: EmitCompareI16(state, inst, OpSLessThanEqual); break; - case IR::Opcode::CompareMaskEqI32: EmitCompareMaskU32(state, inst, OpIEqual); break; - case IR::Opcode::CompareMaskNeI32: EmitCompareMaskU32(state, inst, OpINotEqual); break; - case IR::Opcode::CompareMaskGtI32: EmitCompareMaskU32(state, inst, OpSGreaterThan); break; - case IR::Opcode::CompareMaskGeI32: - EmitCompareMaskU32(state, inst, OpSGreaterThanEqual); - break; - case IR::Opcode::CompareMaskLtI32: EmitCompareMaskU32(state, inst, OpSLessThan); break; - case IR::Opcode::CompareMaskLeI32: EmitCompareMaskU32(state, inst, OpSLessThanEqual); break; - case IR::Opcode::CompareEqU16: EmitCompareU16(state, inst, OpIEqual); break; - case IR::Opcode::CompareNeU16: EmitCompareU16(state, inst, OpINotEqual); break; - case IR::Opcode::CompareGtU16: EmitCompareU16(state, inst, OpUGreaterThan); break; - case IR::Opcode::CompareGeU16: EmitCompareU16(state, inst, OpUGreaterThanEqual); break; - case IR::Opcode::CompareLtU16: EmitCompareU16(state, inst, OpULessThan); break; - case IR::Opcode::CompareLeU16: EmitCompareU16(state, inst, OpULessThanEqual); break; - case IR::Opcode::CompareEqF32: EmitCompareF32(state, inst, OpFOrdEqual); break; - case IR::Opcode::CompareNeF32: EmitCompareF32(state, inst, OpFOrdNotEqual); break; - case IR::Opcode::CompareGtF32: EmitCompareF32(state, inst, OpFOrdGreaterThan); break; - case IR::Opcode::CompareGeF32: EmitCompareF32(state, inst, OpFOrdGreaterThanEqual); break; - case IR::Opcode::CompareLtF32: EmitCompareF32(state, inst, OpFOrdLessThan); break; - case IR::Opcode::CompareLeF32: EmitCompareF32(state, inst, OpFOrdLessThanEqual); break; - case IR::Opcode::CompareOrderedF32: EmitCompareOrderedF32(state, inst, true); break; - case IR::Opcode::CompareUnorderedF32: EmitCompareOrderedF32(state, inst, false); break; - case IR::Opcode::CompareUnordEqF32: EmitCompareF32(state, inst, OpFUnordEqual); break; - case IR::Opcode::CompareUnordNeF32: EmitCompareF32(state, inst, OpFUnordNotEqual); break; - case IR::Opcode::CompareUnordGtF32: EmitCompareF32(state, inst, OpFUnordGreaterThan); break; - case IR::Opcode::CompareUnordGeF32: - EmitCompareF32(state, inst, OpFUnordGreaterThanEqual); - break; - case IR::Opcode::CompareUnordLtF32: EmitCompareF32(state, inst, OpFUnordLessThan); break; - case IR::Opcode::CompareUnordLeF32: - EmitCompareF32(state, inst, OpFUnordLessThanEqual); - break; - case IR::Opcode::CompareClassF32: EmitCompareClassF32(state, inst); break; - case IR::Opcode::CompareEqF16: EmitCompareF16(state, inst, OpFOrdEqual); break; - case IR::Opcode::CompareNeF16: EmitCompareF16(state, inst, OpFOrdNotEqual); break; - case IR::Opcode::CompareGtF16: EmitCompareF16(state, inst, OpFOrdGreaterThan); break; - case IR::Opcode::CompareGeF16: EmitCompareF16(state, inst, OpFOrdGreaterThanEqual); break; - case IR::Opcode::CompareLtF16: EmitCompareF16(state, inst, OpFOrdLessThan); break; - case IR::Opcode::CompareLeF16: EmitCompareF16(state, inst, OpFOrdLessThanEqual); break; - case IR::Opcode::CompareUnordNeF16: EmitCompareF16(state, inst, OpFUnordNotEqual); break; - case IR::Opcode::CompareMaskEqF16: EmitCompareF16(state, inst, OpFOrdEqual); break; - case IR::Opcode::CompareMaskNeF16: EmitCompareF16(state, inst, OpFOrdNotEqual); break; - case IR::Opcode::CompareMaskGtF16: EmitCompareF16(state, inst, OpFOrdGreaterThan); break; - case IR::Opcode::CompareMaskGeF16: - EmitCompareF16(state, inst, OpFOrdGreaterThanEqual); - break; - case IR::Opcode::CompareMaskLtF16: EmitCompareF16(state, inst, OpFOrdLessThan); break; - case IR::Opcode::CompareMaskLeF16: EmitCompareF16(state, inst, OpFOrdLessThanEqual); break; - case IR::Opcode::CompareMaskUnordNeF16: - EmitCompareF16(state, inst, OpFUnordNotEqual); - break; - case IR::Opcode::CompareMaskUnordGeF16: - EmitCompareF16(state, inst, OpFUnordGreaterThanEqual); - break; - case IR::Opcode::CompareMaskEqF32: EmitCompareMaskF32(state, inst, OpFOrdEqual); break; - case IR::Opcode::CompareMaskNeF32: EmitCompareMaskF32(state, inst, OpFOrdNotEqual); break; - case IR::Opcode::CompareMaskGtF32: - EmitCompareMaskF32(state, inst, OpFOrdGreaterThan); - break; - case IR::Opcode::CompareMaskGeF32: - EmitCompareMaskF32(state, inst, OpFOrdGreaterThanEqual); - break; - case IR::Opcode::CompareMaskLtF32: EmitCompareMaskF32(state, inst, OpFOrdLessThan); break; - case IR::Opcode::CompareMaskLeF32: - EmitCompareMaskF32(state, inst, OpFOrdLessThanEqual); - break; - case IR::Opcode::CompareMaskUnordEqF32: - EmitCompareMaskF32(state, inst, OpFUnordEqual); - break; - case IR::Opcode::CompareMaskUnordNeF32: - EmitCompareMaskF32(state, inst, OpFUnordNotEqual); - break; - case IR::Opcode::CompareMaskUnordGtF32: - EmitCompareMaskF32(state, inst, OpFUnordGreaterThan); - break; - case IR::Opcode::CompareMaskUnordGeF32: - EmitCompareMaskF32(state, inst, OpFUnordGreaterThanEqual); - break; - case IR::Opcode::CompareMaskUnordLtF32: - EmitCompareMaskF32(state, inst, OpFUnordLessThan); - break; - case IR::Opcode::CompareMaskUnordLeF32: - EmitCompareMaskF32(state, inst, OpFUnordLessThanEqual); - break; - case IR::Opcode::ConvertU32ToF32: EmitConvertU32ToF32(state, inst); break; - case IR::Opcode::ConvertI32ToF32: EmitConvertI32ToF32(state, inst); break; - case IR::Opcode::ConvertF32ToU32: EmitConvertF32ToU32(state, inst); break; - case IR::Opcode::ConvertF32ToI32: EmitConvertF32ToI32(state, inst); break; - case IR::Opcode::ConvertF32ToF16: EmitConvertF32ToF16(state, inst); break; - case IR::Opcode::ConvertF16ToF32: EmitConvertF16ToF32(state, inst); break; - case IR::Opcode::ConvertU16ToF16: EmitConvertU16ToF16(state, inst); break; - case IR::Opcode::ConvertF16ToU16: EmitConvertF16ToU16(state, inst); break; - case IR::Opcode::ConvertI16ToF16: EmitConvertI16ToF16(state, inst); break; - case IR::Opcode::ConvertF16ToI16: EmitConvertF16ToI16(state, inst); break; - case IR::Opcode::ConvertRoundPlusInfF32ToI32: - EmitConvertRoundPlusInfF32ToI32(state, inst); - break; - case IR::Opcode::ConvertFloorF32ToI32: EmitConvertFloorF32ToI32(state, inst); break; - case IR::Opcode::ConvertI4ToOffsetF32: EmitConvertI4ToOffsetF32(state, inst); break; - case IR::Opcode::LdexpF32: EmitLdexpF32(state, inst); break; - case IR::Opcode::PackF32ToF16Rtz: EmitPackF32ToF16Rtz(state, inst); break; - case IR::Opcode::PackSnorm2x16F32: - EmitPackNormalizedF32(state, inst, GlslPackSnorm2x16); - break; - case IR::Opcode::PackUnorm2x16F32: - EmitPackNormalizedF32(state, inst, GlslPackUnorm2x16); - break; - case IR::Opcode::PackU8F32: EmitPackU8F32(state, inst); break; - case IR::Opcode::PackB32F16: EmitPackB32F16(state, inst); break; - case IR::Opcode::PackedMadI16: - case IR::Opcode::PackedMulLoU16: - case IR::Opcode::PackedAddI16: - case IR::Opcode::PackedSubI16: - case IR::Opcode::PackedLshlrevB16: - case IR::Opcode::PackedLshrrevB16: - case IR::Opcode::PackedAshrrevI16: - case IR::Opcode::PackedMaxI16: - case IR::Opcode::PackedMinI16: - case IR::Opcode::PackedMadU16: - case IR::Opcode::PackedAddU16: - case IR::Opcode::PackedSubU16: - case IR::Opcode::PackedMaxU16: - case IR::Opcode::PackedMinU16: EmitPackedInteger16(state, inst); break; - case IR::Opcode::PackedAddF16: - case IR::Opcode::PackedMulF16: - case IR::Opcode::PackedMinF16: - case IR::Opcode::PackedMaxF16: - case IR::Opcode::PackedFmaF16: EmitPackedF16(state, inst); break; - case IR::Opcode::AddF16: EmitBinaryF16(state, inst, OpFAdd); break; - case IR::Opcode::SubF16: EmitBinaryF16(state, inst, OpFSub); break; - case IR::Opcode::MulF16: EmitBinaryF16(state, inst, OpFMul); break; - case IR::Opcode::MinF16: EmitMinMaxF16(state, inst, false); break; - case IR::Opcode::MaxF16: EmitMinMaxF16(state, inst, true); break; - case IR::Opcode::FmaF16: EmitFmaF16(state, inst); break; - case IR::Opcode::Min3F16: EmitMinMax3F16(state, inst, false); break; - case IR::Opcode::Max3F16: EmitMinMax3F16(state, inst, true); break; - case IR::Opcode::Med3F16: EmitMed3F16(state, inst); break; - case IR::Opcode::MadMixF16: EmitMadMixF16(state, inst); break; - case IR::Opcode::IAddU16: EmitBinaryU16(state, inst, OpIAdd); break; - case IR::Opcode::ISubI16: EmitBinaryU16(state, inst, OpISub); break; - case IR::Opcode::IMinI16: EmitMinMaxU16(state, inst, true, false); break; - case IR::Opcode::IMaxI16: EmitMinMaxU16(state, inst, true, true); break; - case IR::Opcode::UMinU16: EmitMinMaxU16(state, inst, false, false); break; - case IR::Opcode::UMaxU16: EmitMinMaxU16(state, inst, false, true); break; - case IR::Opcode::RcpF16: - case IR::Opcode::SqrtF16: - case IR::Opcode::InverseSqrtF16: - case IR::Opcode::Log2F16: - case IR::Opcode::Exp2F16: - case IR::Opcode::FloorF16: - case IR::Opcode::CeilF16: - case IR::Opcode::TruncF16: - case IR::Opcode::RoundEvenF16: EmitSpecialF16(state, inst); break; - case IR::Opcode::ConvertByteU32ToF32: EmitConvertByteU32ToF32(state, inst); break; - case IR::Opcode::RcpF32: EmitRcpF32(state, inst); break; - case IR::Opcode::FractF32: EmitFloatExtInst(state, inst, GlslFract); break; - case IR::Opcode::TruncF32: EmitFloatExtInst(state, inst, GlslTrunc); break; - case IR::Opcode::CeilF32: EmitFloatExtInst(state, inst, GlslCeil); break; - case IR::Opcode::RoundEvenF32: EmitFloatExtInst(state, inst, GlslRoundEven); break; - case IR::Opcode::FloorF32: EmitFloatExtInst(state, inst, GlslFloor); break; - case IR::Opcode::Exp2F32: EmitFloatExtInst(state, inst, GlslExp2, false, true); break; - case IR::Opcode::Log2F32: EmitFloatExtInst(state, inst, GlslLog2, false, true); break; - case IR::Opcode::InverseSqrtF32: - EmitFloatExtInst(state, inst, GlslInverseSqrt, false, true); - break; - case IR::Opcode::SqrtF32: EmitFloatExtInst(state, inst, GlslSqrt, false, true); break; - case IR::Opcode::SinF32: EmitFloatExtInst(state, inst, GlslSin, true); break; - case IR::Opcode::CosF32: EmitFloatExtInst(state, inst, GlslCos, true); break; - case IR::Opcode::CubeIdF32: EmitCubeF32(state, inst); break; - case IR::Opcode::CubeScF32: EmitCubeF32(state, inst); break; - case IR::Opcode::CubeTcF32: EmitCubeF32(state, inst); break; - case IR::Opcode::CubeMaF32: EmitCubeF32(state, inst); break; - case IR::Opcode::FAddF32: EmitBinaryF32(state, inst, OpFAdd); break; - case IR::Opcode::FSubF32: EmitBinaryF32(state, inst, OpFSub); break; - case IR::Opcode::FMulF32: EmitBinaryF32(state, inst, OpFMul); break; - case IR::Opcode::FMinF32: EmitMinMaxF32(state, inst, false); break; - case IR::Opcode::FMaxF32: EmitMinMaxF32(state, inst, true); break; - case IR::Opcode::FMadF32: EmitMadF32(state, inst); break; - case IR::Opcode::Dot2AccF32F16: EmitDot2AccF32F16(state, inst); break; - case IR::Opcode::FMin3F32: EmitMinMax3F32(state, inst, false); break; - case IR::Opcode::FMax3F32: EmitMinMax3F32(state, inst, true); break; - case IR::Opcode::FMed3F32: EmitMed3F32(state, inst); break; - case IR::Opcode::SLoadDword: EmitSLoadDword(state, inst); break; - case IR::Opcode::SBufferLoadDword: - EmitMemoryLoadU32(state, inst, IR::ResourceKind::ScalarBuffer, 0, 1); - break; - case IR::Opcode::LoadSrtDword: EmitLoadSrtDword(state, inst); break; - case IR::Opcode::BufferLoadUbyte: EmitBufferLoadUbyte(state, inst); break; - case IR::Opcode::BufferLoadSbyte: EmitBufferLoadSbyte(state, inst); break; - case IR::Opcode::BufferLoadUshort: EmitBufferLoadUshort(state, inst); break; - case IR::Opcode::BufferLoadSshort: EmitBufferLoadSshort(state, inst); break; - case IR::Opcode::BufferLoadDword: EmitBufferLoadDword(state, inst); break; - case IR::Opcode::BufferStoreByte: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 8); - }); - break; - case IR::Opcode::BufferStoreShort: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreSubDwordU32(state, inst, IR::ResourceKind::Buffer, 1, - AddressSourceCount(inst, 1), 16); - }); - break; - case IR::Opcode::BufferStoreDword: - EmitGuardedByExec(state, [&]() { EmitBufferStoreDword(state, inst); }); - break; - case IR::Opcode::AtomicSwapU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicExchange); }); - break; - case IR::Opcode::AtomicAddU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicIAdd); }); - break; - case IR::Opcode::AtomicSubU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicISub); }); - break; - case IR::Opcode::AtomicSMinI32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicSMin); }); - break; - case IR::Opcode::AtomicUMinU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicUMin); }); - break; - case IR::Opcode::AtomicSMaxI32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicSMax); }); - break; - case IR::Opcode::AtomicUMaxU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicUMax); }); - break; - case IR::Opcode::AtomicAndU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicAnd); }); - break; - case IR::Opcode::AtomicOrU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicOr); }); - break; - case IR::Opcode::AtomicXorU32: - EmitGuardedByExec(state, [&]() { EmitAtomicU32(state, inst, OpAtomicXor); }); - break; - case IR::Opcode::AtomicFMinF32: - EmitGuardedByExec(state, [&]() { EmitAtomicFMinF32(state, inst); }); - break; - case IR::Opcode::AtomicFMaxF32: - EmitGuardedByExec(state, [&]() { EmitAtomicFMaxF32(state, inst); }); - break; - case IR::Opcode::FlatLoadUbyte: EmitFlatLoadUbyte(state, inst); break; - case IR::Opcode::FlatLoadSbyte: EmitFlatLoadSbyte(state, inst); break; - case IR::Opcode::FlatLoadUshort: EmitFlatLoadUshort(state, inst); break; - case IR::Opcode::FlatLoadSshort: EmitFlatLoadSshort(state, inst); break; - case IR::Opcode::FlatLoadDword: EmitFlatLoadDword(state, inst); break; - case IR::Opcode::FlatStoreByte: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, inst.src_count - 1u, - 8); - }); - break; - case IR::Opcode::FlatStoreShort: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, inst.src_count - 1u, - 16); - }); - break; - case IR::Opcode::FlatStoreDword: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreU32(state, inst, inst.memory.kind, 1, inst.src_count - 1u); - }); - break; - case IR::Opcode::DsReadUbyte: - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 8, false); - break; - case IR::Opcode::DsReadSbyte: - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 8, true); - break; - case IR::Opcode::DsReadUshort: - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 16, false); - break; - case IR::Opcode::DsReadSshort: - EmitMemoryLoadSubDwordU32(state, inst, inst.memory.kind, 0, 1, 16, true); - break; - case IR::Opcode::DsReadB32: EmitMemoryLoadU32(state, inst, inst.memory.kind, 0, 1); break; - case IR::Opcode::DsSwizzleB32: EmitDsSwizzleB32(state, inst); break; - case IR::Opcode::DsConsume: EmitDsAppendConsume(state, inst, OpAtomicISub); break; - case IR::Opcode::DsAppend: EmitDsAppendConsume(state, inst, OpAtomicIAdd); break; - case IR::Opcode::DsReadAddtidB32: EmitDsReadAddtidB32(state, inst); break; - case IR::Opcode::DsMinF32: - EmitGuardedByExec(state, [&]() { EmitDsFloatMinMaxF32(state, inst, false); }); - break; - case IR::Opcode::DsMaxF32: - EmitGuardedByExec(state, [&]() { EmitDsFloatMinMaxF32(state, inst, true); }); - break; - case IR::Opcode::DsWriteByte: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, 1, 8); - }); - break; - case IR::Opcode::DsWriteShort: - EmitGuardedByExec(state, [&]() { - EmitMemoryStoreSubDwordU32(state, inst, inst.memory.kind, 1, 1, 16); - }); - break; - case IR::Opcode::DsWriteB32: - EmitGuardedByExec(state, - [&]() { EmitMemoryStoreU32(state, inst, inst.memory.kind, 1, 1); }); - break; - case IR::Opcode::DsWriteAddtidB32: - EmitGuardedByExec(state, [&]() { EmitDsWriteAddtidB32(state, inst); }); - break; - case IR::Opcode::ImageGetResinfo: EmitImageGetResinfo(state, inst); break; - case IR::Opcode::ImageGetLod: EmitImageGetLod(state, inst); break; - case IR::Opcode::ImageLoad: EmitImageLoad(state, inst); break; - case IR::Opcode::ImageStore: - EmitGuardedByExec(state, [&]() { EmitImageStore(state, inst); }); - break; - case IR::Opcode::ImageSample: EmitImageSample(state, inst); break; - case IR::Opcode::ImageGather4: EmitImageGather4(state, inst); break; - case IR::Opcode::Export: - if (ExportUsesPixelValidMask(state, inst)) { - EmitUpdatePixelValidMask(state); - } - EmitGuardedByExec(state, [&]() { EmitExport(state, inst); }); - break; - default: - if (!state.unsupported_ir_instruction) { - state.unsupported_ir_instruction = true; - state.unsupported_ir_opcode = inst.op; - state.unsupported_ir_pc = inst.pc; - } - break; - } -} - -uint32_t DispatcherTargetValue(EmitterState& state, uint32_t block_id) { - return BlockLabel(state, block_id) != 0 ? block_id : UINT32_MAX; -} - -void EmitDispatcherStoreTarget(EmitterState& state, uint32_t block_id) { - state.builder.AddFunction({OpStore, state.dispatch_pc_variable, - ConstantU32(state, DispatcherTargetValue(state, block_id))}); -} - -void EmitDispatcherExit(EmitterState& state); - -bool RegisterForScalarCode(uint32_t code, IR::Register& reg) { - if (code < 106u) { - reg = {IR::RegisterFile::Scalar, code}; - return true; - } - switch (code) { - case 106u: reg = {IR::RegisterFile::Vcc, 0}; return true; - case 107u: reg = {IR::RegisterFile::Vcc, 1}; return true; - case 124u: reg = {IR::RegisterFile::M0, 0}; return true; - case 126u: reg = {IR::RegisterFile::Exec, 0}; return true; - case 127u: reg = {IR::RegisterFile::Exec, 1}; return true; - default: return false; - } -} - -bool EmitDispatcherLoadScalarCode(EmitterState& state, uint32_t code, uint32_t& value) { - IR::Register reg; - if (!RegisterForScalarCode(code, reg)) { - return false; - } - const auto pointer = PointerForRegister(state, reg); - if (pointer == 0) { - return false; - } - value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, value, pointer}); - return true; -} - -void EmitDispatcherStoreSelectorTarget(EmitterState& state, const CFG::Terminator& term) { - if (term.indirect_selector_code == UINT32_MAX || term.indirect_selector_values.empty() || - term.indirect_selector_values.size() != term.indirect_selector_targets.size()) { - EmitDispatcherExit(state); - return; - } - - uint32_t selector_value = 0; - if (!EmitDispatcherLoadScalarCode(state, term.indirect_selector_code, selector_value)) { - EmitDispatcherExit(state); - return; - } - - uint32_t selected = ConstantU32(state, UINT32_MAX); - for (uint32_t i = 0; i < term.indirect_selector_values.size(); i++) { - const auto match = state.builder.AllocateId(); - state.builder.AddFunction({OpIEqual, state.bool_type, match, selector_value, - ConstantU32(state, term.indirect_selector_values[i])}); - const auto next = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, next, match, - ConstantU32(state, DispatcherTargetValue(state, term.indirect_selector_targets[i])), - selected}); - selected = next; - } - state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected}); - state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label}); -} - -void EmitDispatcherStoreIndirectTarget(EmitterState& state, const CFG::Terminator& term) { - if (term.indirect_selector_code != UINT32_MAX) { - EmitDispatcherStoreSelectorTarget(state, term); - return; - } - - if (term.indirect_pc_sgpr == UINT32_MAX) { - EmitDispatcherExit(state); - return; - } - - const auto pointer = - PointerForRegister(state, IR::Register {IR::RegisterFile::Scalar, term.indirect_pc_sgpr}); - if (pointer == 0) { - EmitDispatcherExit(state); - return; - } - - const auto pc_value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, pc_value, pointer}); - uint32_t selected = ConstantU32(state, UINT32_MAX); - const auto count = std::min(term.indirect_target_pcs.size(), term.indirect_targets.size()); - for (uint32_t i = 0; i < count; i++) { - const auto match = state.builder.AllocateId(); - state.builder.AddFunction({OpIEqual, state.bool_type, match, pc_value, - ConstantU32(state, term.indirect_target_pcs[i])}); - const auto next = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, next, match, - ConstantU32(state, DispatcherTargetValue(state, term.indirect_targets[i])), selected}); - selected = next; - } - state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected}); - state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label}); -} - -void EmitDispatcherExit(EmitterState& state) { - state.builder.AddFunction( - {OpStore, state.dispatch_pc_variable, ConstantU32(state, UINT32_MAX)}); - state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label}); -} - -void EmitDispatcherTerminator(EmitterState& state, const CFG::Terminator& term) { - switch (term.kind) { - case CFG::TerminatorKind::Branch: - if (BlockLabel(state, term.true_block) == 0) { - EmitDispatcherExit(state); - return; - } - EmitDispatcherStoreTarget(state, term.true_block); - state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label}); - return; - case CFG::TerminatorKind::ConditionalBranch: { - const auto condition = EmitBranchCondition(state, term.condition); - const auto selected = state.builder.AllocateId(); - state.builder.AddFunction( - {OpSelect, state.uint_type, selected, condition, - ConstantU32(state, DispatcherTargetValue(state, term.true_block)), - ConstantU32(state, DispatcherTargetValue(state, term.false_block))}); - state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected}); - state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label}); - return; - } - case CFG::TerminatorKind::IndirectBranch: - EmitDispatcherStoreIndirectTarget(state, term); - return; - default: EmitDispatcherExit(state); return; - } -} - -void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) { - state.builder.AddFunction({OpLabel, state.dispatch_header_label}); - const auto pc_value = state.builder.AllocateId(); - state.builder.AddFunction({OpLoad, state.uint_type, pc_value, state.dispatch_pc_variable}); - const auto done = state.builder.AllocateId(); - state.builder.AddFunction( - {OpIEqual, state.bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)}); - state.builder.AddFunction( - {OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone}); - state.builder.AddFunction( - {OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label}); - - state.builder.AddFunction({OpLabel, state.dispatch_select_label}); - state.builder.AddFunction( - {OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone}); - std::vector switch_words = {OpSwitch, pc_value, state.dispatch_default_label}; - for (const auto& block: program.blocks) { - if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) { - continue; - } - const auto label = BlockLabel(state, block.id); - if (label != 0) { - switch_words.push_back(block.id); - switch_words.push_back(label); - } - } - state.builder.AddFunction(switch_words); - - state.builder.AddFunction({OpLabel, state.dispatch_default_label}); - EmitDispatcherExit(state); -} - -size_t BufferLoadGroupSize(const IR::BasicBlock& block, size_t first_index) { - const auto& first = block.instructions[first_index]; - if (first.op != IR::Opcode::BufferLoadDword || first.memory.component_index != 0u || - first.memory.component_count <= 1u) { - return 1u; - } - - size_t count = 1u; - while (first_index + count < block.instructions.size() && - count < first.memory.component_count) { - const auto& next = block.instructions[first_index + count]; - if (next.op != IR::Opcode::BufferLoadDword || next.pc != first.pc || - next.memory.component_index != count || - next.memory.component_count != first.memory.component_count) { - break; - } - count++; - } - return count; -} - -void EmitBlockInstructions(EmitterState& state, const IR::BasicBlock& block) { - for (size_t i = 0; i < block.instructions.size();) { - const auto count = BufferLoadGroupSize(block, i); - if (count > 1u) { - EmitBufferLoadDwordGroup(state, block.instructions.data() + i, - static_cast(count)); - } else { - EmitInstruction(state, block.instructions[i]); - } - i += count; - } -} - -void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) { - for (const auto& block: program.blocks) { - if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) { - continue; - } - state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)}); - EmitBlockInstructions(state, block); - EmitDispatcherTerminator(state, block.terminator); - } -} - -void EmitDispatcherLoopTail(EmitterState& state) { - state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label}); - state.builder.AddFunction({OpBranch, state.dispatch_continue_label}); - state.builder.AddFunction({OpLabel, state.dispatch_continue_label}); - state.builder.AddFunction({OpBranch, state.dispatch_header_label}); - state.builder.AddFunction({OpLabel, state.dispatch_merge_label}); - EmitReturn(state); -} - -void EmitDispatcherFunction(EmitterState& state, const IR::Program& program) { - state.builder.AddFunction({OpStore, state.dispatch_pc_variable, ConstantU32(state, 0)}); - state.builder.AddFunction({OpBranch, state.dispatch_header_label}); - EmitDispatcherSwitch(state, program); - EmitDispatcherBlocks(state, program); - EmitDispatcherLoopTail(state); -} - -void EmitFunction(EmitterState& state, const IR::Program& program) { - state.builder.AddFunction( - {OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type}); - state.builder.AddFunction({OpLabel, state.entry_label}); - - EmitRegisterVariables(state); - EmitComputeInputRegisters(state); - EmitPixelInputRegisters(state); - EmitVertexInputRegisters(state); - EmitStorageBufferOffsets(state); - if (state.dispatcher_fallback) { - EmitDispatcherFunction(state, program); - state.builder.AddFunction({OpFunctionEnd}); - return; - } - - const auto entry_block_label = BlockLabel(state, 0); - if (entry_block_label != 0) { - state.builder.AddFunction({OpBranch, entry_block_label}); - } else { - EmitReturn(state); - } - - for (const auto& block: program.blocks) { - if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) { - continue; - } - state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)}); - EmitBlockInstructions(state, block); - EmitTerminator(state, block.terminator); - } - - state.builder.AddFunction({OpFunctionEnd}); -} - -} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter diff --git a/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp similarity index 80% rename from src/graphics/shader/recompiler/cfg/ShaderCFG.cpp rename to src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp index 58df77f08..ea110e0dd 100644 --- a/src/graphics/shader/recompiler/cfg/ShaderCFG.cpp +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.cpp @@ -1,6 +1,7 @@ -#include "graphics/shader/recompiler/cfg/ShaderCFG.h" +#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" #include +#include #include #include #include @@ -888,6 +889,30 @@ uint32_t AppendSyntheticBranchBlock(Graph& graph, uint32_t target) { return graph.blocks.back().id; } +bool IsolateSemanticLoopHeader(Graph& graph, uint32_t old_header) { + const auto* header = graph.FindBlock(old_header); + if (header == nullptr || header->inst_begin == header->inst_end) { + return false; + } + + const auto predecessors = header->predecessors; + const auto new_header = AppendSyntheticBranchBlock(graph, old_header); + for (auto pred: predecessors) { + auto* block = graph.FindBlock(pred); + if (block != nullptr) { + ReplaceValue(block->successors, old_header, new_header); + ReplaceTerminatorTarget(block->terminator, old_header, new_header); + } + } + if (graph.entry_block == old_header) { + graph.entry_block = new_header; + } + MoveBlockBefore(graph, new_header, old_header); + RebuildPredecessors(graph); + RecomputeAnalyses(graph); + return true; +} + bool IsSyntheticMergeForwarder(const Graph& graph, uint32_t block_id, uint32_t merge) { const auto* block = graph.FindBlock(block_id); return block != nullptr && block->inst_begin == block->inst_end && @@ -913,6 +938,43 @@ bool IsInsideLoopConstruct(const Graph& graph, const NaturalLoop& loop, uint32_t graph.Dominates(loop.header, block_id) && !graph.Dominates(loop.merge, block_id); } +bool IsLoopControlGateway(const Graph& graph, const NaturalLoop& loop, uint32_t block_id) { + const auto* block = graph.FindBlock(block_id); + if (block == nullptr || block->terminator.kind != TerminatorKind::ConditionalBranch) { + return false; + } + const auto is_control_target = [&](uint32_t target) { + return target == loop.merge || target == loop.continue_block; + }; + return is_control_target(block->terminator.true_block) && + is_control_target(block->terminator.false_block); +} + +uint32_t FindSelectionMerge(const Graph& graph, const BasicBlock& block) { + const auto global_merge = graph.FindNearestCommonPostDominator(block.terminator.true_block, + block.terminator.false_block); + const auto* loop = FindInnermostContainingLoop(graph, block.id); + if (loop == nullptr) { + return global_merge; + } + + // A selection can join at the loop-control gateway even when another arm breaks directly to + // the enclosing loop merge. Global post-dominance sees that break and incorrectly expands the + // selection through the backedge. + const auto true_target = block.terminator.true_block; + const auto false_target = block.terminator.false_block; + if (IsLoopControlGateway(graph, *loop, true_target) && graph.Dominates(block.id, true_target) && + IsInsideLoopConstruct(graph, *loop, false_target)) { + return true_target; + } + if (IsLoopControlGateway(graph, *loop, false_target) && + graph.Dominates(block.id, false_target) && + IsInsideLoopConstruct(graph, *loop, true_target)) { + return false_target; + } + return global_merge; +} + bool IsInnermostLoopControlConditional(const Graph& graph, const BasicBlock& block) { if (block.terminator.kind != TerminatorKind::ConditionalBranch) { return false; @@ -993,22 +1055,7 @@ bool CanonicalizeNaturalLoops(Graph& graph, std::string* error) { continue; } - const auto old_header = loop.header; - const auto predecessors = header->predecessors; - const auto new_header = AppendSyntheticBranchBlock(graph, old_header); - for (auto pred: predecessors) { - auto* block = graph.FindBlock(pred); - if (block != nullptr) { - ReplaceValue(block->successors, old_header, new_header); - ReplaceTerminatorTarget(block->terminator, old_header, new_header); - } - } - if (graph.entry_block == old_header) { - graph.entry_block = new_header; - } - MoveBlockBefore(graph, new_header, old_header); - RebuildPredecessors(graph); - RecomputeAnalyses(graph); + IsolateSemanticLoopHeader(graph, loop.header); changed = true; break; } @@ -1101,12 +1148,13 @@ bool SplitOneLoopMerge(Graph& graph) { std::vector SelectionRegion(const Graph& graph, const BasicBlock& header, uint32_t merge) { std::vector region; - std::vector pending = {header.terminator.true_block, - header.terminator.false_block}; + std::vector pending = {header.terminator.true_block, header.terminator.false_block}; + const auto* loop = FindInnermostContainingLoop(graph, header.id); while (!pending.empty()) { const auto block_id = pending.back(); pending.pop_back(); - if (block_id == merge || Contains(region, block_id)) { + if (block_id == merge || Contains(region, block_id) || + (loop != nullptr && (block_id == loop->merge || block_id == loop->continue_block))) { continue; } const auto* block = graph.FindBlock(block_id); @@ -1120,80 +1168,7 @@ std::vector SelectionRegion(const Graph& graph, const BasicBlock& head return region; } -bool DuplicateSelectionRegion(Graph& graph, uint32_t header_id, uint32_t merge, - const std::vector& region, uint32_t block_budget) { - std::vector cloned_blocks; - for (auto block_id: region) { - if (!graph.Dominates(header_id, block_id)) { - cloned_blocks.push_back(block_id); - } - } - if (cloned_blocks.empty() || graph.FindBlock(header_id) == nullptr || header_id >= merge || - graph.blocks.size() + cloned_blocks.size() + 1u > block_budget) { - return false; - } - - const auto first_clone = static_cast(graph.blocks.size()); - std::map clones; - for (uint32_t i = 0; i < cloned_blocks.size(); i++) { - clones.emplace(cloned_blocks[i], first_clone + i); - } - - for (auto block_id: cloned_blocks) { - BasicBlock clone = *graph.FindBlock(block_id); - clone.id = clones.at(block_id); - clone.predecessors.clear(); - clone.dominators.clear(); - clone.post_dominators.clear(); - graph.blocks.push_back(std::move(clone)); - } - - const auto remap_block = [&](BasicBlock& block) { - const auto remap_target = [&](uint32_t& target) { - if (const auto it = clones.find(target); it != clones.end()) { - target = it->second; - } - }; - for (auto& successor: block.successors) { - remap_target(successor); - } - remap_target(block.terminator.true_block); - remap_target(block.terminator.false_block); - remap_target(block.terminator.merge_block); - remap_target(block.terminator.continue_block); - for (auto& target: block.terminator.indirect_targets) { - remap_target(target); - } - }; - for (auto block_id: region) { - const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id; - remap_block(*graph.FindBlock(owned_id)); - } - - const auto private_merge = AppendSyntheticBranchBlock(graph, merge); - auto& header = *graph.FindBlock(header_id); - remap_block(header); - - for (auto block_id: region) { - const auto owned_id = clones.contains(block_id) ? clones.at(block_id) : block_id; - auto* block = graph.FindBlock(owned_id); - if (block != nullptr) { - ReplaceValue(block->successors, merge, private_merge); - ReplaceTerminatorTarget(block->terminator, merge, private_merge); - } - } - ReplaceValue(header.successors, merge, private_merge); - ReplaceTerminatorTarget(header.terminator, merge, private_merge); - - for (uint32_t i = 0; i <= cloned_blocks.size(); i++) { - MoveBlockBefore(graph, first_clone + i, merge + i); - } - RebuildPredecessors(graph); - RecomputeAnalyses(graph); - return true; -} - -bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) { +bool SplitOneSelectionMerge(Graph& graph, std::string* error) { std::vector loop_headers; loop_headers.reserve(graph.natural_loops.size()); for (const auto& loop: graph.natural_loops) { @@ -1211,19 +1186,22 @@ bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) { continue; } - const auto merge = graph.FindNearestCommonPostDominator(block->terminator.true_block, - block->terminator.false_block); + const auto merge = FindSelectionMerge(graph, *block); if (merge == UINT32_MAX || graph.FindBlock(merge) == nullptr) { continue; } - const auto region = SelectionRegion(graph, *block, merge); - if (std::any_of(region.begin(), region.end(), - [&](uint32_t member) { return !graph.Dominates(block_id, member); })) { - if (graph.natural_loops.empty() && - DuplicateSelectionRegion(graph, block_id, merge, region, block_budget)) { - return true; - } - continue; + const auto region = SelectionRegion(graph, *block, merge); + const auto external = std::find_if(region.begin(), region.end(), [&](uint32_t member) { + return !graph.Dominates(block_id, member); + }); + if (external != region.end()) { + SetFailure( + graph, FailureKind::StructuredControlFlow, block_id, + fmt::format("selection header block {} has externally entered region block {}; " + "semantic block cloning is disabled", + block_id, *external), + error); + return false; } const auto construct_blocks = DominatedBlocks(graph, block_id, merge); const auto force_split = MergeLeavesContainingLoop(graph, block_id, merge); @@ -1236,13 +1214,11 @@ bool SplitOneSelectionMerge(Graph& graph, uint32_t block_budget) { bool SplitSharedMergeBlocks(Graph& graph, std::string* error) { const auto original_block_count = static_cast(graph.blocks.size()); - const auto split_budget = std::max( - 16u, std::min(128u, original_block_count * 4u)); - const auto block_budget = std::max( - 32u, std::min(512u, original_block_count * 8u)); + const auto split_budget = + std::max(16u, std::min(128u, original_block_count * 4u)); for (uint32_t splits = 0; splits < split_budget; splits++) { - if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, block_budget)) { - return true; + if (!SplitOneLoopMerge(graph) && !SplitOneSelectionMerge(graph, error)) { + return !graph.unsupported; } RebuildPredecessors(graph); RecomputeAnalyses(graph); @@ -1339,6 +1315,182 @@ uint32_t Graph::FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_ return common.empty() ? UINT32_MAX : common.front(); } +namespace { + +struct GotoRouteBlocks { + uint32_t before = UINT32_MAX; + std::array blocks {}; +}; + +void PlaceGotoRouteBlocks(Graph& graph, uint32_t original_block_count, + const GotoRouteBlocks& route) { + std::vector old_blocks = std::move(graph.blocks); + std::vector new_blocks; + new_blocks.reserve(old_blocks.size()); + for (uint32_t block_id = 0; block_id < original_block_count; block_id++) { + if (block_id == route.before) { + for (const auto route_block: route.blocks) { + new_blocks.push_back(std::move(old_blocks[route_block])); + } + } + new_blocks.push_back(std::move(old_blocks[block_id])); + } + + std::vector id_map(new_blocks.size(), UINT32_MAX); + for (uint32_t block_id = 0; block_id < new_blocks.size(); block_id++) { + id_map[new_blocks[block_id].id] = block_id; + } + + graph.blocks = std::move(new_blocks); + graph.entry_block = RemapId(graph.entry_block, id_map); + for (auto& block: graph.blocks) { + block.id = RemapId(block.id, id_map); + RemapIds(block.predecessors, id_map); + RemapIds(block.successors, id_map); + RemapIds(block.dominators, id_map); + RemapIds(block.post_dominators, id_map); + block.terminator.true_block = RemapId(block.terminator.true_block, id_map); + block.terminator.false_block = RemapId(block.terminator.false_block, id_map); + block.terminator.merge_block = RemapId(block.terminator.merge_block, id_map); + block.terminator.continue_block = RemapId(block.terminator.continue_block, id_map); + } +} + +uint32_t AppendGotoSelectBlock(Graph& graph, uint32_t route_variable, uint32_t true_target, + uint32_t false_target) { + const auto* target = graph.FindBlock(false_target); + BasicBlock block; + block.id = static_cast(graph.blocks.size()); + block.start_pc = target != nullptr ? target->start_pc : 0u; + block.end_pc = block.start_pc; + block.inst_begin = target != nullptr ? target->inst_begin : 0u; + block.inst_end = block.inst_begin; + block.successors = {true_target, false_target}; + block.terminator.kind = TerminatorKind::ConditionalBranch; + block.terminator.condition = BranchCondition::GotoVariable; + block.terminator.true_block = true_target; + block.terminator.false_block = false_target; + block.terminator.goto_variable = route_variable; + graph.blocks.push_back(std::move(block)); + return graph.blocks.back().id; +} + +uint32_t AppendGotoSetBlock(Graph& graph, uint32_t route_variable, bool value, uint32_t target) { + const auto block_id = AppendSyntheticBranchBlock(graph, target); + auto* block = graph.FindBlock(block_id); + block->terminator.goto_variable = route_variable; + block->terminator.goto_value = value ? 1 : 0; + return block_id; +} + +// Turn H0 -> shared/H1, H1 -> shared/other into nested selections whose empty +// forwarding blocks set one typed SSA route value. Guest semantic blocks remain unique. +bool RouteOneSharedArm(Graph& graph, uint32_t original_block_count, uint32_t outer_id, + uint32_t route_variable, GotoRouteBlocks& route) { + auto* outer = graph.FindBlock(outer_id); + if (outer == nullptr || outer->inst_begin == outer->inst_end || + outer->terminator.kind != TerminatorKind::ConditionalBranch || + std::ranges::any_of(graph.natural_loops, [outer_id](const auto& loop) { + return Contains(loop.body_blocks, outer_id); + })) { + return false; + } + + for (const bool inner_is_true: {true, false}) { + const auto inner_id = + inner_is_true ? outer->terminator.true_block : outer->terminator.false_block; + const auto shared = + inner_is_true ? outer->terminator.false_block : outer->terminator.true_block; + const auto* inner = graph.FindBlock(inner_id); + if (inner == nullptr || inner->inst_begin == inner->inst_end || + inner->predecessors != std::vector {outer_id} || + inner->terminator.kind != TerminatorKind::ConditionalBranch) { + continue; + } + + const bool true_is_shared = inner->terminator.true_block == shared; + const bool false_is_shared = inner->terminator.false_block == shared; + if (true_is_shared == false_is_shared) { + continue; + } + const auto other = + true_is_shared ? inner->terminator.false_block : inner->terminator.true_block; + if (other == outer_id || other == inner_id || other == shared) { + continue; + } + const auto first_arm = std::min(shared, other); + if (first_arm >= original_block_count || outer_id >= inner_id || inner_id >= first_arm) { + continue; + } + const auto* shared_block = graph.FindBlock(shared); + const auto* other_block = graph.FindBlock(other); + if (shared_block == nullptr || other_block == nullptr || + shared_block->terminator.kind != TerminatorKind::Branch || + other_block->terminator.kind != TerminatorKind::Branch || + shared_block->successors.size() != 1u || + shared_block->successors != other_block->successors) { + continue; + } + const auto in_loop = [&](uint32_t block_id) { + return std::ranges::any_of(graph.natural_loops, [&](const auto& loop) { + return Contains(loop.body_blocks, block_id); + }); + }; + if (in_loop(inner_id) || in_loop(shared) || in_loop(other)) { + continue; + } + + const auto route_select = AppendGotoSelectBlock(graph, route_variable, other, shared); + const auto inner_merge = AppendSyntheticBranchBlock(graph, route_select); + const auto outer_shared = AppendGotoSetBlock(graph, route_variable, false, route_select); + const auto inner_shared = AppendGotoSetBlock(graph, route_variable, false, inner_merge); + const auto inner_other = AppendGotoSetBlock(graph, route_variable, true, inner_merge); + + outer = graph.FindBlock(outer_id); + auto* mutable_inner = graph.FindBlock(inner_id); + outer->terminator.true_block = inner_is_true ? inner_id : outer_shared; + outer->terminator.false_block = inner_is_true ? outer_shared : inner_id; + outer->successors = {outer->terminator.true_block, outer->terminator.false_block}; + mutable_inner->terminator.true_block = true_is_shared ? inner_shared : inner_other; + mutable_inner->terminator.false_block = true_is_shared ? inner_other : inner_shared; + mutable_inner->successors = {mutable_inner->terminator.true_block, + mutable_inner->terminator.false_block}; + route = {first_arm, {inner_shared, inner_other, inner_merge, outer_shared, route_select}}; + return true; + } + return false; +} + +bool RouteTerminalSharedSelectionArm(Graph& graph) { + if (graph.irreducible) { + return false; + } + const auto original_block_count = static_cast(graph.blocks.size()); + GotoRouteBlocks route; + bool found = false; + for (uint32_t block_id = 0; block_id < original_block_count; block_id++) { + GotoRouteBlocks next; + if (RouteOneSharedArm(graph, original_block_count, block_id, 0u, next)) { + if (found) { + // Multiple interacting overlaps remain on the dispatcher path. + return false; + } + found = true; + route = next; + RebuildPredecessors(graph); + } + } + if (!found) { + return false; + } + PlaceGotoRouteBlocks(graph, original_block_count, route); + RebuildPredecessors(graph); + RecomputeAnalyses(graph); + return true; +} + +} // namespace + bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error) { graph = {}; @@ -1554,9 +1706,14 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro ComputeComponents(graph); if (indirect_setpc) { - graph.irreducible = true; - graph.unsupported = false; - graph.failure_kind = FailureKind::IrreducibleControlFlow; + graph.irreducible = true; + graph.unsupported = false; + graph.failure_kind = FailureKind::IrreducibleControlFlow; + const auto indirect = + std::find_if(graph.blocks.begin(), graph.blocks.end(), [](const auto& block) { + return block.terminator.kind == TerminatorKind::IndirectBranch; + }); + graph.failure_block = indirect != graph.blocks.end() ? indirect->id : graph.entry_block; graph.unsupported_reason = "indirect S_SETPC_B64 jump table requires dispatcher fallback"; } @@ -1567,7 +1724,9 @@ bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* erro return true; } -bool Structurize(Graph& graph, std::string* error) { +namespace { + +bool StructurizeImpl(Graph& graph, std::string* error) { if (graph.unsupported || graph.irreducible) { if (graph.unsupported_reason.empty()) { graph.unsupported_reason = "unsupported CFG"; @@ -1625,8 +1784,7 @@ bool Structurize(Graph& graph, std::string* error) { continue; } - const auto merge = graph.FindNearestCommonPostDominator(block.terminator.true_block, - block.terminator.false_block); + const auto merge = FindSelectionMerge(graph, block); if (merge == UINT32_MAX) { SetFailure(graph, FailureKind::StructuredControlFlow, block.id, fmt::format("conditional block {} has no structured merge", block.id), @@ -1643,6 +1801,57 @@ bool Structurize(Graph& graph, std::string* error) { return true; } +} // namespace + +bool Structurize(Graph& graph, std::string* error) { + Graph original = graph; + if (StructurizeImpl(graph, error)) { + return true; + } + if (!RouteTerminalSharedSelectionArm(original)) { + return false; + } + Graph failed_graph = std::move(graph); + std::string failed_error = error != nullptr ? *error : std::string {}; + graph = std::move(original); + if (error != nullptr) { + error->clear(); + } + if (StructurizeImpl(graph, error)) { + return true; + } + graph = std::move(failed_graph); + if (error != nullptr) { + *error = std::move(failed_error); + } + return false; +} + +bool IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error) { + const auto loop = + std::find_if(graph.natural_loops.begin(), graph.natural_loops.end(), + [header_id](const auto& value) { return value.header == header_id; }); + if (loop == graph.natural_loops.end()) { + SetFailure(graph, FailureKind::StructuredControlFlow, header_id, + fmt::format("block {} is not a natural loop header", header_id), error); + return false; + } + + ClearStructuredTerminators(graph); + if (!IsolateSemanticLoopHeader(graph, header_id)) { + SetFailure(graph, FailureKind::StructuredControlFlow, header_id, + fmt::format("loop header block {} has no semantic instructions", header_id), + error); + return false; + } + + graph.unsupported = false; + graph.failure_kind = FailureKind::None; + graph.failure_block = UINT32_MAX; + graph.unsupported_reason.clear(); + return Structurize(graph, error); +} + std::string BranchConditionToString(BranchCondition condition) { switch (condition) { case BranchCondition::Always: return "always"; @@ -1652,6 +1861,7 @@ std::string BranchConditionToString(BranchCondition condition) { case BranchCondition::VccNonZero: return "vccnz"; case BranchCondition::ExecZero: return "execz"; case BranchCondition::ExecNonZero: return "execnz"; + case BranchCondition::GotoVariable: return "goto_variable"; default: return "unknown"; } } @@ -1670,19 +1880,6 @@ std::string FailureKindToString(FailureKind kind) { } } -std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage, - const std::string& reason) { - const auto* block = graph.FindBlock(block_id); - const char* stage_text = stage != nullptr ? stage : "unknown"; - if (block == nullptr) { - return fmt::format("stage={} block={} reason={}", stage_text, block_id, reason.c_str()); - } - return fmt::format("stage={} block={} pc=0x{:08x}..0x{:08x} preds={} succs={} reason={}", - stage_text, block->id, block->start_pc, block->end_pc, - static_cast(block->predecessors.size()), - static_cast(block->successors.size()), reason.c_str()); -} - std::string GraphToString(const Graph& graph) { std::string text; text += @@ -1706,14 +1903,16 @@ std::string GraphToString(const Graph& graph) { VectorToString(block.post_dominators).c_str()); text += fmt::format( " terminator={} condition={} true={} false={} merge={} continue={} loop_header={} " - "indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}]\n", + "indirect_sgpr={} indirect_selector={} indirect_targets=[{}] selector_values=[{}] " + "goto_variable={} goto_value={}\n", static_cast(block.terminator.kind), BranchConditionToString(block.terminator.condition).c_str(), block.terminator.true_block, block.terminator.false_block, block.terminator.merge_block, block.terminator.continue_block, block.terminator.loop_header ? 1u : 0u, block.terminator.indirect_pc_sgpr, block.terminator.indirect_selector_code, VectorToString(block.terminator.indirect_targets).c_str(), - VectorToString(block.terminator.indirect_selector_values).c_str()); + VectorToString(block.terminator.indirect_selector_values).c_str(), + block.terminator.goto_variable, block.terminator.goto_value); } for (const auto& edge: graph.back_edges) { diff --git a/src/graphics/shader/recompiler/cfg/ShaderCFG.h b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h similarity index 92% rename from src/graphics/shader/recompiler/cfg/ShaderCFG.h rename to src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h index afd4a7602..868ff5ef5 100644 --- a/src/graphics/shader/recompiler/cfg/ShaderCFG.h +++ b/src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h @@ -3,7 +3,7 @@ #include "common/common.h" #include "common/stringUtils.h" -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include @@ -17,6 +17,7 @@ enum class BranchCondition { VccNonZero, ExecZero, ExecNonZero, + GotoVariable, Unknown }; @@ -46,7 +47,9 @@ struct Terminator { std::vector indirect_targets; std::vector indirect_selector_values; std::vector indirect_selector_targets; - bool loop_header = false; + uint32_t goto_variable = UINT32_MAX; + int32_t goto_value = -1; + bool loop_header = false; }; struct BasicBlock { @@ -107,10 +110,9 @@ struct Graph { bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error); bool Structurize(Graph& graph, std::string* error); +bool IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error); std::string BranchConditionToString(BranchCondition condition); std::string FailureKindToString(FailureKind kind); -std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage, - const std::string& reason); std::string GraphToString(const Graph& graph); } // namespace Libs::Graphics::ShaderRecompiler::CFG diff --git a/src/graphics/shader/recompiler/decompiler/ExportOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ExportOps.cpp similarity index 95% rename from src/graphics/shader/recompiler/decompiler/ExportOps.cpp rename to src/graphics/shader/recompiler/frontend/decode/ExportOps.cpp index 2e450e744..02ae830b3 100644 --- a/src/graphics/shader/recompiler/decompiler/ExportOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ExportOps.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/decompiler/ExportOps.h" +#include "graphics/shader/recompiler/frontend/decode/ExportOps.h" #include diff --git a/src/graphics/shader/recompiler/decompiler/ExportOps.h b/src/graphics/shader/recompiler/frontend/decode/ExportOps.h similarity index 87% rename from src/graphics/shader/recompiler/decompiler/ExportOps.h rename to src/graphics/shader/recompiler/frontend/decode/ExportOps.h index 3ccbedebe..f0faee33a 100644 --- a/src/graphics/shader/recompiler/decompiler/ExportOps.h +++ b/src/graphics/shader/recompiler/frontend/decode/ExportOps.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" namespace Libs::Graphics::ShaderRecompiler::Decoder { diff --git a/src/graphics/shader/recompiler/decompiler/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp similarity index 99% rename from src/graphics/shader/recompiler/decompiler/ImageOps.cpp rename to src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 28e668dcc..9e1c29b6c 100644 --- a/src/graphics/shader/recompiler/decompiler/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/decompiler/ImageOps.h" +#include "graphics/shader/recompiler/frontend/decode/ImageOps.h" #include #include diff --git a/src/graphics/shader/recompiler/decompiler/ImageOps.h b/src/graphics/shader/recompiler/frontend/decode/ImageOps.h similarity index 88% rename from src/graphics/shader/recompiler/decompiler/ImageOps.h rename to src/graphics/shader/recompiler/frontend/decode/ImageOps.h index 3f80d498c..887789941 100644 --- a/src/graphics/shader/recompiler/decompiler/ImageOps.h +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" namespace Libs::Graphics::ShaderRecompiler::Decoder { diff --git a/src/graphics/shader/recompiler/decompiler/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp similarity index 99% rename from src/graphics/shader/recompiler/decompiler/MemoryOps.cpp rename to src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 5c2bf2198..bd1e4b459 100644 --- a/src/graphics/shader/recompiler/decompiler/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/decompiler/MemoryOps.h" +#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h" #include #include diff --git a/src/graphics/shader/recompiler/decompiler/MemoryOps.h b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.h similarity index 93% rename from src/graphics/shader/recompiler/decompiler/MemoryOps.h rename to src/graphics/shader/recompiler/frontend/decode/MemoryOps.h index 5a2d5e514..b7d6217ed 100644 --- a/src/graphics/shader/recompiler/decompiler/MemoryOps.h +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" namespace Libs::Graphics::ShaderRecompiler::Decoder { diff --git a/src/graphics/shader/recompiler/decompiler/OpcodeTable.h b/src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h similarity index 94% rename from src/graphics/shader/recompiler/decompiler/OpcodeTable.h rename to src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h index 5177cc95a..e49581d4f 100644 --- a/src/graphics/shader/recompiler/decompiler/OpcodeTable.h +++ b/src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include diff --git a/src/graphics/shader/recompiler/decompiler/ScalarAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp similarity index 98% rename from src/graphics/shader/recompiler/decompiler/ScalarAluOps.cpp rename to src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp index 50de42a0e..5d39f9d8d 100644 --- a/src/graphics/shader/recompiler/decompiler/ScalarAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp @@ -1,6 +1,6 @@ -#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h" +#include "graphics/shader/recompiler/frontend/decode/ScalarAluOps.h" -#include "graphics/shader/recompiler/decompiler/OpcodeTable.h" +#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h" namespace Libs::Graphics::ShaderRecompiler::Decoder { namespace { diff --git a/src/graphics/shader/recompiler/decompiler/ScalarAluOps.h b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.h similarity index 93% rename from src/graphics/shader/recompiler/decompiler/ScalarAluOps.h rename to src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.h index 12b983d49..4ec91c14a 100644 --- a/src/graphics/shader/recompiler/decompiler/ScalarAluOps.h +++ b/src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" namespace Libs::Graphics::ShaderRecompiler::Decoder { diff --git a/src/graphics/shader/recompiler/decompiler/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp similarity index 99% rename from src/graphics/shader/recompiler/decompiler/ShaderDecoder.cpp rename to src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index da29067a3..b192346c5 100644 --- a/src/graphics/shader/recompiler/decompiler/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -1,10 +1,10 @@ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" -#include "graphics/shader/recompiler/decompiler/ExportOps.h" -#include "graphics/shader/recompiler/decompiler/ImageOps.h" -#include "graphics/shader/recompiler/decompiler/MemoryOps.h" -#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h" -#include "graphics/shader/recompiler/decompiler/VectorAluOps.h" +#include "graphics/shader/recompiler/frontend/decode/ExportOps.h" +#include "graphics/shader/recompiler/frontend/decode/ImageOps.h" +#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h" +#include "graphics/shader/recompiler/frontend/decode/ScalarAluOps.h" +#include "graphics/shader/recompiler/frontend/decode/VectorAluOps.h" #include #include @@ -556,6 +556,7 @@ std::string OpcodeToString(Opcode opcode) { case Opcode::VCvtF32Ubyte2: return "v_cvt_f32_ubyte2"; case Opcode::VCvtF32Ubyte3: return "v_cvt_f32_ubyte3"; case Opcode::VRcpF32: return "v_rcp_f32"; + case Opcode::VRcpIflagF32: return "v_rcp_iflag_f32"; case Opcode::VFractF32: return "v_fract_f32"; case Opcode::VTruncF32: return "v_trunc_f32"; case Opcode::VCeilF32: return "v_ceil_f32"; diff --git a/src/graphics/shader/recompiler/decompiler/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h similarity index 92% rename from src/graphics/shader/recompiler/decompiler/ShaderDecoder.h rename to src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index 1da5b489a..9cca50dde 100644 --- a/src/graphics/shader/recompiler/decompiler/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -154,6 +154,7 @@ enum class Opcode { VCvtF32Ubyte2, VCvtF32Ubyte3, VRcpF32, + VRcpIflagF32, VFractF32, VTruncF32, VCeilF32, @@ -587,27 +588,29 @@ constexpr uint32_t MaxInstructionRawWords = 5u; constexpr uint32_t MaxImageNsaAddressComponents = 12u; struct Operand { - OperandKind kind = OperandKind::Unknown; - uint32_t value = 0; - int32_t signed_val = 0; - float float_val = 0.0f; - uint32_t reg = 0; - uint32_t sdwa_sel = 6; - uint32_t sdwa_dst_unused = 0; - uint32_t omod = 0; - uint32_t dpp_ctrl = 0; - uint32_t dpp_row_mask = 0xf; - uint32_t dpp_bank_mask = 0xf; - bool sdwa_sext = false; - bool dpp_fetch_inactive = false; - bool dpp_bound_ctrl = false; - bool op_sel = false; - bool op_sel_hi = false; - bool negate = false; - bool negate_hi = false; - bool absolute = false; - bool clamp = false; - bool dpp = false; + OperandKind kind = OperandKind::Unknown; + uint32_t value = 0; + int32_t signed_val = 0; + float float_val = 0.0f; + uint32_t reg = 0; + uint32_t sdwa_sel = 6; + // Native 16-bit destinations use the same selector fields internally but preserve the + // untouched half. Real SDWA encodings overwrite this with their decoded DST_U value. + uint32_t sdwa_dst_unused = 2; + uint32_t omod = 0; + uint32_t dpp_ctrl = 0; + uint32_t dpp_row_mask = 0xf; + uint32_t dpp_bank_mask = 0xf; + bool sdwa_sext = false; + bool dpp_fetch_inactive = false; + bool dpp_bound_ctrl = false; + bool op_sel = false; + bool op_sel_hi = false; + bool negate = false; + bool negate_hi = false; + bool absolute = false; + bool clamp = false; + bool dpp = false; }; struct Instruction { diff --git a/src/graphics/shader/recompiler/decompiler/VectorAluOps.cpp b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp similarity index 99% rename from src/graphics/shader/recompiler/decompiler/VectorAluOps.cpp rename to src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp index 6ca9db97e..d84b948a4 100644 --- a/src/graphics/shader/recompiler/decompiler/VectorAluOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.cpp @@ -1,6 +1,6 @@ -#include "graphics/shader/recompiler/decompiler/VectorAluOps.h" +#include "graphics/shader/recompiler/frontend/decode/VectorAluOps.h" -#include "graphics/shader/recompiler/decompiler/OpcodeTable.h" +#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h" #include @@ -111,7 +111,7 @@ constexpr OpcodeMap VOP1_OPS[] = { {0x24u, Opcode::VFloorF32}, {0x25u, Opcode::VExpF32}, {0x27u, Opcode::VLogF32}, - {0x2bu, Opcode::VRcpF32}, + {0x2bu, Opcode::VRcpIflagF32}, {0x2eu, Opcode::VRsqF32}, {0x33u, Opcode::VSqrtF32}, {0x35u, Opcode::VSinF32}, @@ -157,7 +157,7 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPS[] = { {0x24u, Opcode::VFloorF32}, {0x25u, Opcode::VExpF32}, {0x27u, Opcode::VLogF32}, - {0x2bu, Opcode::VRcpF32}, + {0x2bu, Opcode::VRcpIflagF32}, {0x2eu, Opcode::VRsqF32}, {0x33u, Opcode::VSqrtF32}, {0x35u, Opcode::VSinF32}, @@ -397,6 +397,7 @@ bool IsVop1FloatSourceOpcode(Opcode opcode) { case Opcode::VCvtFlrI32F32: case Opcode::VCvtF16F32: case Opcode::VRcpF32: + case Opcode::VRcpIflagF32: case Opcode::VFractF32: case Opcode::VTruncF32: case Opcode::VCeilF32: @@ -743,6 +744,7 @@ bool IsVop1FloatResultOpcode(Opcode opcode) { case Opcode::VCvtF32Ubyte2: case Opcode::VCvtF32Ubyte3: case Opcode::VRcpF32: + case Opcode::VRcpIflagF32: case Opcode::VFractF32: case Opcode::VTruncF32: case Opcode::VCeilF32: diff --git a/src/graphics/shader/recompiler/decompiler/VectorAluOps.h b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.h similarity index 94% rename from src/graphics/shader/recompiler/decompiler/VectorAluOps.h rename to src/graphics/shader/recompiler/frontend/decode/VectorAluOps.h index e1c01ab0b..64cbe510f 100644 --- a/src/graphics/shader/recompiler/decompiler/VectorAluOps.h +++ b/src/graphics/shader/recompiler/frontend/decode/VectorAluOps.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_ -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" namespace Libs::Graphics::ShaderRecompiler::Decoder { diff --git a/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp new file mode 100644 index 000000000..19779d564 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Attribute.cpp @@ -0,0 +1,35 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) { + const auto index = static_cast(value_program.export_info.size()); + value_program.export_info.push_back(inst.export_info); + return {.index = index, .pc = inst.pc}; +} + +bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) { + if (inst.op == IR::Opcode::LoadInputF32) { + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::GetAttribute, + {IR::Value(inst.input_info.attr), IR::Value(inst.input_info.chan)})); + return true; + } + if (inst.op == IR::Opcode::Export) { + std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), + IR::Value(0u)}; + for (uint32_t index = 0; index < std::min(inst.src_count, 4u); index++) { + components[index] = ReadRawU32(PlainOperand(inst.src[index])); + } + const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, + {components[0], components[1], components[2], components[3]}); + ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst)); + return true; + } + return false; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Compare.cpp b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp new file mode 100644 index 000000000..0a3d0d964 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Compare.cpp @@ -0,0 +1,195 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +bool Translator::TranslateIntegerCompare(const IR::Instruction& inst) { + IR::ValueOpcode opcode {}; + IR::Type type = IR::Type::U32; + switch (inst.op) { + case IR::Opcode::CompareFalse: + WriteCompareResult(inst.dst, IR::U1(IR::Value(false))); + return true; + case IR::Opcode::CompareTrue: + WriteCompareResult(inst.dst, IR::U1(IR::Value(true))); + return true; + case IR::Opcode::CompareEqU32: + case IR::Opcode::CompareMaskEqU32: + case IR::Opcode::CompareEqI32: + case IR::Opcode::CompareMaskEqI32: opcode = IR::ValueOpcode::IEqual32; break; + case IR::Opcode::CompareNeU32: + case IR::Opcode::CompareMaskNeU32: + case IR::Opcode::CompareNeI32: + case IR::Opcode::CompareMaskNeI32: opcode = IR::ValueOpcode::INotEqual32; break; + case IR::Opcode::CompareGtU32: + case IR::Opcode::CompareMaskGtU32: opcode = IR::ValueOpcode::UGreaterThan32; break; + case IR::Opcode::CompareGeU32: + case IR::Opcode::CompareMaskGeU32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; + case IR::Opcode::CompareLtU32: + case IR::Opcode::CompareMaskLtU32: opcode = IR::ValueOpcode::ULessThan32; break; + case IR::Opcode::CompareLeU32: + case IR::Opcode::CompareMaskLeU32: opcode = IR::ValueOpcode::ULessThanEqual32; break; + case IR::Opcode::CompareGtI32: + case IR::Opcode::CompareMaskGtI32: opcode = IR::ValueOpcode::SGreaterThan32; break; + case IR::Opcode::CompareGeI32: + case IR::Opcode::CompareMaskGeI32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break; + case IR::Opcode::CompareLtI32: + case IR::Opcode::CompareMaskLtI32: opcode = IR::ValueOpcode::SLessThan32; break; + case IR::Opcode::CompareLeI32: + case IR::Opcode::CompareMaskLeI32: opcode = IR::ValueOpcode::SLessThanEqual32; break; + case IR::Opcode::CompareEqU64: + opcode = IR::ValueOpcode::IEqual64; + type = IR::Type::U64; + break; + case IR::Opcode::CompareNeU64: + opcode = IR::ValueOpcode::INotEqual64; + type = IR::Type::U64; + break; + default: return false; + } + const auto lhs = ReadOperand(inst.src[0], type); + const auto rhs = ReadOperand(inst.src[1], type); + WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs}))); + return true; +} + +bool Translator::TranslateInteger16Compare(const IR::Instruction& inst) { + IR::ValueOpcode opcode {}; + bool signed_value = false; + switch (inst.op) { + case IR::Opcode::CompareEqU16: + case IR::Opcode::CompareEqI16: opcode = IR::ValueOpcode::IEqual32; break; + case IR::Opcode::CompareNeU16: + case IR::Opcode::CompareNeI16: opcode = IR::ValueOpcode::INotEqual32; break; + case IR::Opcode::CompareGtU16: opcode = IR::ValueOpcode::UGreaterThan32; break; + case IR::Opcode::CompareGeU16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break; + case IR::Opcode::CompareLtU16: opcode = IR::ValueOpcode::ULessThan32; break; + case IR::Opcode::CompareLeU16: opcode = IR::ValueOpcode::ULessThanEqual32; break; + case IR::Opcode::CompareGtI16: + opcode = IR::ValueOpcode::SGreaterThan32; + signed_value = true; + break; + case IR::Opcode::CompareGeI16: + opcode = IR::ValueOpcode::SGreaterThanEqual32; + signed_value = true; + break; + case IR::Opcode::CompareLtI16: + opcode = IR::ValueOpcode::SLessThan32; + signed_value = true; + break; + case IR::Opcode::CompareLeI16: + opcode = IR::ValueOpcode::SLessThanEqual32; + signed_value = true; + break; + default: return false; + } + if (inst.op == IR::Opcode::CompareEqI16 || inst.op == IR::Opcode::CompareNeI16) { + signed_value = true; + } + WriteCompareResult(inst.dst, + IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src[0], signed_value), + ReadU16AsU32(inst.src[1], signed_value)}))); + return true; +} + +bool Translator::TranslateFloatCompare(const IR::Instruction& inst) { + IR::ValueOpcode opcode {}; + bool half = false; + switch (inst.op) { + case IR::Opcode::CompareEqF32: + case IR::Opcode::CompareMaskEqF32: opcode = IR::ValueOpcode::FPOrdEqual32; break; + case IR::Opcode::CompareNeF32: + case IR::Opcode::CompareMaskNeF32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break; + case IR::Opcode::CompareGtF32: + case IR::Opcode::CompareMaskGtF32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break; + case IR::Opcode::CompareGeF32: + case IR::Opcode::CompareMaskGeF32: opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; break; + case IR::Opcode::CompareLtF32: + case IR::Opcode::CompareMaskLtF32: opcode = IR::ValueOpcode::FPOrdLessThan32; break; + case IR::Opcode::CompareLeF32: + case IR::Opcode::CompareMaskLeF32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break; + case IR::Opcode::CompareUnordEqF32: + case IR::Opcode::CompareMaskUnordEqF32: opcode = IR::ValueOpcode::FPUnordEqual32; break; + case IR::Opcode::CompareUnordNeF32: + case IR::Opcode::CompareMaskUnordNeF32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break; + case IR::Opcode::CompareUnordGtF32: + case IR::Opcode::CompareMaskUnordGtF32: + opcode = IR::ValueOpcode::FPUnordGreaterThan32; + break; + case IR::Opcode::CompareUnordGeF32: + case IR::Opcode::CompareMaskUnordGeF32: + opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32; + break; + case IR::Opcode::CompareUnordLtF32: + case IR::Opcode::CompareMaskUnordLtF32: opcode = IR::ValueOpcode::FPUnordLessThan32; break; + case IR::Opcode::CompareUnordLeF32: + case IR::Opcode::CompareMaskUnordLeF32: + opcode = IR::ValueOpcode::FPUnordLessThanEqual32; + break; + case IR::Opcode::CompareEqF16: + case IR::Opcode::CompareMaskEqF16: + opcode = IR::ValueOpcode::FPOrdEqual32; + half = true; + break; + case IR::Opcode::CompareNeF16: + case IR::Opcode::CompareMaskNeF16: + opcode = IR::ValueOpcode::FPOrdNotEqual32; + half = true; + break; + case IR::Opcode::CompareGtF16: + case IR::Opcode::CompareMaskGtF16: + opcode = IR::ValueOpcode::FPOrdGreaterThan32; + half = true; + break; + case IR::Opcode::CompareGeF16: + case IR::Opcode::CompareMaskGeF16: + opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; + half = true; + break; + case IR::Opcode::CompareLtF16: + case IR::Opcode::CompareMaskLtF16: + opcode = IR::ValueOpcode::FPOrdLessThan32; + half = true; + break; + case IR::Opcode::CompareLeF16: + case IR::Opcode::CompareMaskLeF16: + opcode = IR::ValueOpcode::FPOrdLessThanEqual32; + half = true; + break; + case IR::Opcode::CompareUnordNeF16: + case IR::Opcode::CompareMaskUnordNeF16: + opcode = IR::ValueOpcode::FPUnordNotEqual32; + half = true; + break; + case IR::Opcode::CompareMaskUnordGeF16: + opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32; + half = true; + break; + case IR::Opcode::CompareOrderedF32: + case IR::Opcode::CompareUnorderedF32: { + const auto lhs = IR::F32(ReadOperand(inst.src[0], IR::Type::F32)); + const auto rhs = IR::F32(ReadOperand(inst.src[1], IR::Type::F32)); + const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})), + IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs}))); + const auto result = + inst.op == IR::Opcode::CompareUnorderedF32 ? unordered : ir.LogicalNot(unordered); + WriteCompareResult(inst.dst, result); + return true; + } + case IR::Opcode::CompareClassF32: { + const auto value = ReadOperand(inst.src[0], IR::Type::F32); + const auto mask = ReadOperand(inst.src[1], IR::Type::U32); + WriteCompareResult(inst.dst, + IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask}))); + return true; + } + default: return false; + } + const auto lhs = + half ? IR::Value(ReadF16AsF32(inst.src[0])) : ReadOperand(inst.src[0], IR::Type::F32); + const auto rhs = + half ? IR::Value(ReadF16AsF32(inst.src[1])) : ReadOperand(inst.src[1], IR::Type::F32); + WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs}))); + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Control.cpp b/src/graphics/shader/recompiler/frontend/translate/Control.cpp new file mode 100644 index 000000000..b168d344d --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Control.cpp @@ -0,0 +1,260 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +bool Translator::TranslateStateOperation(const IR::Instruction& inst) { + const auto zero = IR::U32(IR::Value(0u)); + switch (inst.op) { + case IR::Opcode::SaveexecB32: + case IR::Opcode::SaveexecB64: { + const auto old_low = ir.GetExecLo(); + const auto old_high = ir.GetExecHi(); + const auto src_low = ReadRawU32(inst.src[0]); + const auto src_high = inst.op == IR::Opcode::SaveexecB64 + ? ReadRawU32(OffsetOperand(inst.src[0], 1)) + : IR::U32(IR::Value(0u)); + const auto apply = [&](IR::U32 old_value, IR::U32 src_value) { + switch (inst.saveexec_mode) { + case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value); + case IR::SaveexecMode::Andn1: + return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value)); + case IR::SaveexecMode::Orn2: + return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value); + } + return old_value; + }; + const auto result_low = apply(old_low, src_low); + const auto result_high = + inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high; + if (inst.op == IR::Opcode::SaveexecB64) { + WriteOperand(inst.dst, ir.PackUint2x32(old_low, old_high)); + } else { + WriteOperand(inst.dst, old_low); + } + ir.SetExecLo(result_low); + ir.SetExecHi(result_high); + ir.SetExec(ThreadBit(result_low, result_high)); + const auto nonzero = inst.op == IR::Opcode::SaveexecB64 + ? ir.BitwiseOr(result_low, result_high) + : result_low; + ir.SetScc(ir.INotEqual(nonzero, zero)); + return true; + } + case IR::Opcode::ScalarAddCarryU32: + case IR::Opcode::IAddCarryU32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const auto carry_in = ConditionBit(inst.src[2]); + const auto add0 = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs}); + const auto partial = ir.CompositeExtract(add0, 0); + const auto carry0 = ir.CompositeExtract(add0, 1); + const auto add1 = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in}); + const auto result = ir.CompositeExtract(add1, 0); + const auto carry1 = ir.CompositeExtract(add1, 1); + const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), IR::U32(IR::Value(0u))); + WriteOperand(inst.dst, result); + if (inst.op == IR::Opcode::ScalarAddCarryU32) { + ir.SetScc(carry); + } else { + WriteMask(inst.dst2, carry); + } + return true; + } + case IR::Opcode::ScalarSubBorrowU32: + case IR::Opcode::ISubBorrowU32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const auto result = ir.ISub(lhs, rhs); + const auto borrow = ir.UGreaterThan(rhs, lhs); + WriteOperand(inst.dst, result); + if (inst.op == IR::Opcode::ScalarSubBorrowU32) { + ir.SetScc(borrow); + } else { + WriteMask(inst.dst2, borrow); + } + return true; + } + case IR::Opcode::ScalarSubBorrowCarryU32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const auto borrow_in = ConditionBit(inst.src[2]); + const auto partial = ir.ISub(lhs, rhs); + const auto result = ir.ISub(partial, borrow_in); + const auto borrow0 = ir.UGreaterThan(rhs, lhs); + const auto borrow1 = ir.UGreaterThan(borrow_in, partial); + WriteOperand(inst.dst, result); + ir.SetScc(ir.LogicalOr(borrow0, borrow1)); + return true; + } + case IR::Opcode::ScalarSignedAddOverflowI32: + case IR::Opcode::ScalarSignedSubOverflowI32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const bool subtract = inst.op == IR::Opcode::ScalarSignedSubOverflowI32; + const auto result = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs); + const auto shift = IR::U32(IR::Value(31u)); + const auto lhs_sign = ir.ShiftRightLogical(lhs, shift); + const auto rhs_sign = ir.ShiftRightLogical(rhs, shift); + const auto out_sign = ir.ShiftRightLogical(result, shift); + const auto inputs = + subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign); + const auto changed = ir.INotEqual(lhs_sign, out_sign); + WriteOperand(inst.dst, result); + ir.SetScc(ir.LogicalAnd(inputs, changed)); + return true; + } + case IR::Opcode::ScalarShiftLeftAddCarryU32: { + const auto lhs = ReadU32(inst.src[0]); + const auto shift = ReadU32(inst.src[1]); + const auto rhs = ReadU32(inst.src[2]); + const auto shifted = ir.ShiftLeftLogical(lhs, shift); + const auto result = ir.IAdd(shifted, rhs); + const auto add_carry = ir.ULessThan(result, shifted); + const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift); + const auto shifted_out = ir.ShiftRightLogical(lhs, inverse_shift); + const auto shift_carry = ir.INotEqual(shifted_out, zero); + WriteOperand(inst.dst, result); + ir.SetScc(ir.LogicalOr(add_carry, shift_carry)); + return true; + } + default: return false; + } +} + +bool Translator::TranslateControlOperation(const IR::Instruction& inst) { + switch (inst.op) { + case IR::Opcode::ControlNop: ir.Emit(IR::ValueOpcode::ControlNop); return true; + case IR::Opcode::Waitcnt: ir.Emit(IR::ValueOpcode::Waitcnt); return true; + case IR::Opcode::Barrier: ir.Emit(IR::ValueOpcode::Barrier); return true; + case IR::Opcode::Sendmsg: ir.Emit(IR::ValueOpcode::Sendmsg); return true; + case IR::Opcode::TtraceData: ir.Emit(IR::ValueOpcode::TtraceData); return true; + case IR::Opcode::InstPrefetch: ir.Emit(IR::ValueOpcode::InstPrefetch); return true; + default: return false; + } +} + +bool Translator::TranslateMove(const IR::Instruction& inst) { + switch (inst.op) { + case IR::Opcode::MoveU32: + if (inst.src[0].kind == IR::OperandKind::Register && + (inst.src[0].reg.file == IR::RegisterFile::Exec || + inst.src[0].reg.file == IR::RegisterFile::Vcc) && + inst.dst.kind == IR::OperandKind::Register && + (inst.dst.reg.file == IR::RegisterFile::Exec || + inst.dst.reg.file == IR::RegisterFile::Vcc)) { + WriteMask(inst.dst, ReadMask(inst.src[0])); + } else { + WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U32)); + } + return true; + case IR::Opcode::MoveF32Bits: + WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32)); + return true; + case IR::Opcode::MoveU64: { + const bool scalar_pair = inst.dst.kind == IR::OperandKind::Register && + inst.src[0].kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar && + inst.src[0].reg.file == IR::RegisterFile::Scalar; + WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U64)); + if (scalar_pair) { + ir.SetThreadBitScalarReg( + static_cast(inst.dst.reg.index), + ir.GetThreadBitScalarReg(static_cast(inst.src[0].reg.index))); + } + return true; + } + case IR::Opcode::WqmB64: { + if (!current_per_invocation_masks && inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + const auto result = + ExpandWholeQuadMask(IR::U64(ReadOperand(inst.src[0], IR::Type::U64))); + WriteOperand(inst.dst, result); + ir.SetScc(IR::U1( + ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}))); + return true; + } + const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])})); + WriteMask64(inst.dst, result); + ir.SetScc(result); + return true; + } + default: return false; + } +} + +bool Translator::TranslateLaneOperation(const IR::Instruction& inst) { + const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u)); + switch (inst.op) { + case IR::Opcode::MoveRelSourceU32: { + EXIT_IF(inst.src[0].kind != IR::OperandKind::Register || + inst.src[0].reg.file != IR::RegisterFile::Vector); + const auto base = inst.src[0].reg.index; + const auto m0 = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu))); + auto selected = ir.GetVectorReg(static_cast(base)); + for (uint32_t index = base + 1u; index < current_vector_limit; index++) { + const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); + selected = + ir.Select(match, ir.GetVectorReg(static_cast(index)), selected); + } + WriteOperand(inst.dst, selected); + return true; + } + case IR::Opcode::MoveRelDestU32: { + EXIT_IF(inst.dst.kind != IR::OperandKind::Register || + inst.dst.reg.file != IR::RegisterFile::Vector); + const auto base = inst.dst.reg.index; + const auto value = ReadU32(inst.src[0]); + const auto m0 = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu))); + for (uint32_t index = base; index < current_vector_limit; index++) { + const auto reg = static_cast(index); + const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base))); + const auto write = ir.LogicalAnd(ir.GetExec(), match); + ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg))); + } + return true; + } + case IR::Opcode::ReadFirstLaneU32: { + const auto result = + ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src[0]), ir.GetExec()}); + WriteOperand(inst.dst, result); + return true; + } + case IR::Opcode::ReadLaneU32: { + const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask); + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src[0]), lane})); + return true; + } + case IR::Opcode::WriteLaneU32: { + EXIT_IF(inst.dst.kind != IR::OperandKind::Register || + inst.dst.reg.file != IR::RegisterFile::Vector); + const auto reg = static_cast(inst.dst.reg.index); + const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask); + const auto result = IR::U32(ir.Emit( + IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src[0]), lane})); + // V_WRITELANE_B32 ignores EXEC; bypass the ordinary VGPR destination path. + ir.SetVectorReg(reg, result); + return true; + } + case IR::Opcode::Permlane16B32: + case IR::Opcode::Permlanex16B32: { + const IR::PermlaneFlags flags { + .x16 = inst.op == IR::Opcode::Permlanex16B32, + .fetch_inactive = inst.dst.op_sel, + .bound_control = inst.dst.op_sel_hi, + }; + const auto result = ir.Emit( + IR::ValueOpcode::Permlane16U32, + {ReadU32(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]), ir.GetExec()}, + flags); + auto dst = inst.dst; + dst.op_sel = false; + dst.op_sel_hi = false; + WriteOperand(dst, result); + return true; + } + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Convert.cpp b/src/graphics/shader/recompiler/frontend/translate/Convert.cpp new file mode 100644 index 000000000..498692eef --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Convert.cpp @@ -0,0 +1,158 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +IR::F32 Translator::SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value) { + return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value})); +} + +IR::U32 Translator::ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, + uint32_t upper_result) { + const auto zero = IR::F32(IR::Value::F32(0.0f)); + const auto nan = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value})); + const auto low = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, zero})); + const auto high = IR::U1( + ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)})); + const auto truncated = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value})); + const auto safe_low = SelectF32(ir.LogicalOr(nan, low), zero, truncated); + const auto safe = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low); + const auto converted = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32F32, {safe})); + return ir.Select(high, IR::U32(IR::Value(upper_result)), converted); +} + +IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound, + float safe_upper, uint32_t lower_result, + uint32_t upper_result) { + const auto nan = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value})); + const auto low = IR::U1( + ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, IR::Value::F32(lower_bound)})); + const auto high = IR::U1( + ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)})); + const auto truncated = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value})); + const auto safe_low = SelectF32(low, IR::F32(IR::Value::F32(lower_bound)), truncated); + const auto safe_high = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low); + const auto safe = SelectF32(nan, IR::F32(IR::Value::F32(0.0f)), safe_high); + const auto converted = IR::U32(ir.Emit(IR::ValueOpcode::ConvertS32F32, {safe})); + const auto clamped_high = ir.Select(high, IR::U32(IR::Value(upper_result)), converted); + const auto clamped = ir.Select(low, IR::U32(IR::Value(lower_result)), clamped_high); + return ir.Select(nan, IR::U32(IR::Value(0u)), clamped); +} + +bool Translator::TranslateConversion(const IR::Instruction& inst) { + const auto f32 = [&](uint32_t index) { + return IR::F32(ReadOperand(inst.src[index], IR::Type::F32)); + }; + const auto u32 = [&](uint32_t index) { return ReadU32(inst.src[index]); }; + const auto convert_u32 = [&](IR::F32 value) { + return ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu); + }; + const auto convert_i32 = [&](IR::F32 value) { + return ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f, + 0x80000000u, 0x7fffffffu); + }; + + switch (inst.op) { + case IR::Opcode::ConvertByteU32ToF32: { + const auto index = + inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm & 3u : 0u; + const auto byte = + ir.BitwiseAnd(ir.ShiftRightLogical(u32(0), IR::U32(IR::Value(index * 8u))), + IR::U32(IR::Value(0xffu))); + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte})); + return true; + } + case IR::Opcode::ConvertU32ToF32: + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)})); + return true; + case IR::Opcode::ConvertI32ToF32: + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)})); + return true; + case IR::Opcode::ConvertF32ToU32: WriteOperand(inst.dst, convert_u32(f32(0))); return true; + case IR::Opcode::ConvertF32ToI32: WriteOperand(inst.dst, convert_i32(f32(0))); return true; + case IR::Opcode::ConvertF32ToF16: WriteF16(inst.dst, f32(0)); return true; + case IR::Opcode::ConvertF16ToF32: + WriteOperand(inst.dst, ReadF16AsF32(inst.src[0])); + return true; + case IR::Opcode::ConvertU16ToF16: { + const auto value = IR::F32( + ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src[0], false)})); + WriteF16(inst.dst, value); + return true; + } + case IR::Opcode::ConvertI16ToF16: { + const auto value = + IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src[0], true)})); + WriteF16(inst.dst, value); + return true; + } + case IR::Opcode::ConvertF16ToU16: { + const auto converted = + ConvertF32ToU32Saturated(ReadF16AsF32(inst.src[0]), 65536.0f, 65535.0f, 0xffffu); + WriteU16(inst.dst, converted); + return true; + } + case IR::Opcode::ConvertF16ToI16: { + const auto converted = ConvertF32ToI32Saturated( + ReadF16AsF32(inst.src[0]), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu); + WriteU16(inst.dst, ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu)))); + return true; + } + case IR::Opcode::ConvertRoundPlusInfF32ToI32: { + const auto biased = + IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {f32(0), IR::Value::F32(0.5f)})); + const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased})); + WriteOperand(inst.dst, convert_i32(rounded)); + return true; + } + case IR::Opcode::ConvertFloorF32ToI32: { + const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {f32(0)})); + WriteOperand(inst.dst, convert_i32(rounded)); + return true; + } + case IR::Opcode::ConvertI4ToOffsetF32: { + const auto nibble = IR::U32( + ir.Emit(IR::ValueOpcode::BitFieldSExtract, {u32(0), IR::Value(0u), IR::Value(4u)})); + const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble})); + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)})); + return true; + } + case IR::Opcode::PackF32ToF16Rtz: + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)})); + return true; + case IR::Opcode::PackSnorm2x16F32: + case IR::Opcode::PackUnorm2x16F32: { + const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {f32(0), f32(1)}); + const auto opcode = inst.op == IR::Opcode::PackSnorm2x16F32 + ? IR::ValueOpcode::PackSnorm2x16 + : IR::ValueOpcode::PackUnorm2x16; + WriteOperand(inst.dst, ir.Emit(opcode, {pair})); + return true; + } + case IR::Opcode::PackU8F32: { + const auto byte = ConvertF32ToU32Saturated(f32(0), 255.0f, 255.0f, 255u); + const auto index = ir.BitwiseAnd(u32(1), IR::U32(IR::Value(3u))); + const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u))); + const auto mask = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift); + const auto base = ir.BitwiseAnd(u32(2), ir.BitwiseNot(mask)); + WriteOperand(inst.dst, ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift))); + return true; + } + case IR::Opcode::PackB32F16: { + const auto low = ReadF16LaneBits(inst.src[0], false); + const auto high = + ir.ShiftLeftLogical(ReadF16LaneBits(inst.src[1], false), IR::U32(IR::Value(16u))); + WriteOperand(inst.dst, ir.BitwiseOr(low, high)); + return true; + } + default: return false; + } +} + +IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) { + const auto mask = IR::U32(IR::Value(0xffffu)); + return ir.BitwiseOr(ir.BitwiseAnd(low, mask), + ir.ShiftLeftLogical(ir.BitwiseAnd(high, mask), IR::U32(IR::Value(16u)))); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp new file mode 100644 index 000000000..f1238031a --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp @@ -0,0 +1,40 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) { + if (current_per_invocation_masks) { + return TranslatePerInvocationU64Mask(inst); + } + return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 + ? TranslateSimpleInteger(inst) + : TranslateComposedInteger(inst); +} + +bool Translator::TranslateInstruction(const IR::Instruction& inst, + const BufferAddressValues* address_snapshot) { + switch (IR::GetOpcodeInfo(inst.op).lowering_class) { + case IR::LoweringClass::Control: return TranslateControlOperation(inst); + case IR::LoweringClass::Move: return TranslateMove(inst); + case IR::LoweringClass::Lane: return TranslateLaneOperation(inst); + case IR::LoweringClass::State: return TranslateStateOperation(inst); + case IR::LoweringClass::Memory: return TranslateMemoryOperation(inst, address_snapshot); + case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst); + case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst); + case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst); + case IR::LoweringClass::FloatCompare: return TranslateFloatCompare(inst); + case IR::LoweringClass::Conversion: return TranslateConversion(inst); + case IR::LoweringClass::Integer16: return TranslateInteger16Operation(inst); + case IR::LoweringClass::PackedInteger16: return TranslatePackedInteger16(inst); + case IR::LoweringClass::PackedFloat16: return TranslatePackedFloat16(inst); + case IR::LoweringClass::Float16: return TranslateFloat16Operation(inst); + case IR::LoweringClass::Float: return TranslateFloatOperation(inst); + case IR::LoweringClass::U64Mask: return TranslateU64MaskOperation(inst); + case IR::LoweringClass::SimpleInteger: return TranslateSimpleInteger(inst); + case IR::LoweringClass::ComposedInteger: return TranslateComposedInteger(inst); + case IR::LoweringClass::ExtendedInteger: return TranslateExtendedInteger(inst); + } + return false; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Float.cpp b/src/graphics/shader/recompiler/frontend/translate/Float.cpp new file mode 100644 index 000000000..f55cb1527 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Float.cpp @@ -0,0 +1,212 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) { + IR::ValueOpcode opcode; + switch (inst.op) { + case IR::Opcode::PackedAddF16: opcode = IR::ValueOpcode::FPAdd32; break; + case IR::Opcode::PackedMulF16: opcode = IR::ValueOpcode::FPMul32; break; + case IR::Opcode::PackedMinF16: opcode = IR::ValueOpcode::FPMin32; break; + case IR::Opcode::PackedMaxF16: opcode = IR::ValueOpcode::FPMax32; break; + case IR::Opcode::PackedFmaF16: opcode = IR::ValueOpcode::FPFma32; break; + default: return false; + } + const auto translate_lane = [&](bool high) { + const auto lhs = ReadF16LaneAsF32(inst.src[0], high); + const auto rhs = ReadF16LaneAsF32(inst.src[1], high); + IR::F32 result; + if (inst.src_count == 3u) { + result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high)})); + } else { + result = IR::F32(ir.Emit(opcode, {lhs, rhs})); + } + return ApplyF32ResultModifiers(inst.dst, result); + }; + auto raw = inst.dst; + raw.omod = 0u; + raw.clamp = false; + auto result = PackHalf2x16(translate_lane(false), translate_lane(true)); + if (inst.op == IR::Opcode::PackedMinF16 || inst.op == IR::Opcode::PackedMaxF16) { + const auto quiet_snan = [&](const IR::Operand& operand, bool high) { + const auto bits = ReadU16LaneAsU32(operand, high, false); + const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u))); + const auto payload = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu))); + const auto snan = ir.LogicalAnd(ir.IEqual(exponent, IR::U32(IR::Value(0x7c00u))), + ir.INotEqual(payload, IR::U32(IR::Value(0u)))); + return std::pair {snan, ir.BitwiseOr(bits, IR::U32(IR::Value(0x0200u)))}; + }; + const auto override_lane = [&](bool high) { + const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src[0], high); + const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src[1], high); + const auto normal = high ? ir.ShiftRightLogical(result, IR::U32(IR::Value(16u))) + : ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))); + return ir.Select(lhs_snan, lhs_quiet, ir.Select(rhs_snan, rhs_quiet, normal)); + }; + result = PackU16Lanes(override_lane(false), override_lane(true)); + } + WriteOperand(raw, result); + return true; +} + +bool Translator::TranslateFloat16Operation(const IR::Instruction& inst) { + IR::ValueOpcode opcode; + switch (inst.op) { + case IR::Opcode::AddF16: opcode = IR::ValueOpcode::FPAdd32; break; + case IR::Opcode::SubF16: opcode = IR::ValueOpcode::FPSub32; break; + case IR::Opcode::MulF16: opcode = IR::ValueOpcode::FPMul32; break; + case IR::Opcode::MinF16: opcode = IR::ValueOpcode::FPMin32; break; + case IR::Opcode::MaxF16: opcode = IR::ValueOpcode::FPMax32; break; + case IR::Opcode::FmaF16: + case IR::Opcode::MadMixF16: opcode = IR::ValueOpcode::FPFma32; break; + case IR::Opcode::RcpF16: opcode = IR::ValueOpcode::FPRecip32; break; + case IR::Opcode::SqrtF16: opcode = IR::ValueOpcode::FPSqrt; break; + case IR::Opcode::InverseSqrtF16: opcode = IR::ValueOpcode::FPRecipSqrt32; break; + case IR::Opcode::Log2F16: opcode = IR::ValueOpcode::FPLog2; break; + case IR::Opcode::Exp2F16: opcode = IR::ValueOpcode::FPExp2; break; + case IR::Opcode::FloorF16: opcode = IR::ValueOpcode::FPFloor32; break; + case IR::Opcode::CeilF16: opcode = IR::ValueOpcode::FPCeil32; break; + case IR::Opcode::TruncF16: opcode = IR::ValueOpcode::FPTrunc32; break; + case IR::Opcode::RoundEvenF16: opcode = IR::ValueOpcode::FPRoundEven32; break; + case IR::Opcode::Min3F16: opcode = IR::ValueOpcode::FPMinTri32; break; + case IR::Opcode::Max3F16: opcode = IR::ValueOpcode::FPMaxTri32; break; + case IR::Opcode::Med3F16: opcode = IR::ValueOpcode::FPMedTri32; break; + default: return false; + } + std::array args; + for (uint32_t index = 0; index < inst.src_count; index++) { + args[index] = inst.op == IR::Opcode::MadMixF16 ? IR::Value(ReadMixF32(inst.src[index])) + : IR::Value(ReadF16AsF32(inst.src[index])); + } + IR::F32 result; + switch (inst.src_count) { + case 1: result = IR::F32(ir.Emit(opcode, {args[0]})); break; + case 2: result = IR::F32(ir.Emit(opcode, {args[0], args[1]})); break; + case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break; + default: EXIT("invalid half-float source count: %u", inst.src_count); + } + WriteF16(inst.dst, result); + return true; +} + +bool Translator::TranslateFloatOperation(const IR::Instruction& inst) { + IR::ValueOpcode opcode {}; + switch (inst.op) { + case IR::Opcode::RcpF32: opcode = IR::ValueOpcode::FPRecip32; break; + case IR::Opcode::RcpIflagF32: opcode = IR::ValueOpcode::FPRecipIFlag32; break; + case IR::Opcode::FractF32: opcode = IR::ValueOpcode::FPFract32; break; + case IR::Opcode::TruncF32: opcode = IR::ValueOpcode::FPTrunc32; break; + case IR::Opcode::CeilF32: opcode = IR::ValueOpcode::FPCeil32; break; + case IR::Opcode::RoundEvenF32: opcode = IR::ValueOpcode::FPRoundEven32; break; + case IR::Opcode::FloorF32: opcode = IR::ValueOpcode::FPFloor32; break; + case IR::Opcode::Exp2F32: opcode = IR::ValueOpcode::FPExp2; break; + case IR::Opcode::Log2F32: opcode = IR::ValueOpcode::FPLog2; break; + case IR::Opcode::InverseSqrtF32: opcode = IR::ValueOpcode::FPRecipSqrt32; break; + case IR::Opcode::SqrtF32: opcode = IR::ValueOpcode::FPSqrt; break; + case IR::Opcode::SinF32: opcode = IR::ValueOpcode::FPSin; break; + case IR::Opcode::CosF32: opcode = IR::ValueOpcode::FPCos; break; + case IR::Opcode::FAddF32: opcode = IR::ValueOpcode::FPAdd32; break; + case IR::Opcode::FSubF32: opcode = IR::ValueOpcode::FPSub32; break; + case IR::Opcode::FMulF32: opcode = IR::ValueOpcode::FPMul32; break; + case IR::Opcode::FMinF32: opcode = IR::ValueOpcode::FPMin32; break; + case IR::Opcode::FMaxF32: opcode = IR::ValueOpcode::FPMax32; break; + case IR::Opcode::FMadF32: opcode = IR::ValueOpcode::FPFma32; break; + case IR::Opcode::FMin3F32: opcode = IR::ValueOpcode::FPMinTri32; break; + case IR::Opcode::FMax3F32: opcode = IR::ValueOpcode::FPMaxTri32; break; + case IR::Opcode::FMed3F32: opcode = IR::ValueOpcode::FPMedTri32; break; + case IR::Opcode::LdexpF32: opcode = IR::ValueOpcode::FPLdexp; break; + default: break; + } + if (opcode != IR::ValueOpcode {}) { + std::array args; + for (uint32_t index = 0; index < inst.src_count; index++) { + const auto arg_type = IR::ArgTypeOf(opcode, index); + args[index] = + arg_type == IR::Type::F32 + ? (inst.op == IR::Opcode::FMadF32 ? IR::Value(ReadMixF32(inst.src[index])) + : ReadOperand(inst.src[index], IR::Type::F32)) + : ReadOperand(inst.src[index], arg_type); + } + IR::Value result; + switch (inst.src_count) { + case 1: result = ir.Emit(opcode, {args[0]}); break; + case 2: result = ir.Emit(opcode, {args[0], args[1]}); break; + case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break; + default: EXIT("invalid floating-point source count: %u", inst.src_count); + } + WriteOperand(inst.dst, result); + return true; + } + + const auto select_f32 = [&](IR::U1 condition, IR::F32 true_value, IR::F32 false_value) { + return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value})); + }; + if (inst.op == IR::Opcode::Dot2AccF32F16) { + auto a = inst.src[0]; + a.op_sel = false; + a.op_sel_hi = true; + auto b = inst.src[1]; + b.op_sel = false; + b.op_sel_hi = true; + const auto a_lo = ReadF16LaneAsF32(a, false); + const auto a_hi = ReadF16LaneAsF32(a, true); + const auto b_lo = ReadF16LaneAsF32(b, false); + const auto b_hi = ReadF16LaneAsF32(b, true); + const auto acc = ReadMixF32(inst.src[2]); + const auto lo = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc})); + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo})); + return true; + } + if (inst.op != IR::Opcode::CubeIdF32 && inst.op != IR::Opcode::CubeScF32 && + inst.op != IR::Opcode::CubeTcF32 && inst.op != IR::Opcode::CubeMaF32) { + return false; + } + + const auto x = ReadMixF32(inst.src[0]); + const auto y = ReadMixF32(inst.src[1]); + const auto z = ReadMixF32(inst.src[2]); + const auto nx = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {x})); + const auto ny = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {y})); + const auto nz = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {z})); + const auto ax = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {x})); + const auto ay = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {y})); + const auto az = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {z})); + const auto z_face = + ir.LogicalAnd(IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {az, ax})), + IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {az, ay}))); + const auto y_face = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {ay, ax})); + const auto x_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {x, IR::Value::F32(0.0f)})); + const auto y_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {y, IR::Value::F32(0.0f)})); + const auto z_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {z, IR::Value::F32(0.0f)})); + const auto select_face = [&](IR::F32 x_value, IR::F32 y_value, IR::F32 z_value) { + return select_f32(z_face, z_value, select_f32(y_face, y_value, x_value)); + }; + IR::F32 result; + switch (inst.op) { + case IR::Opcode::CubeIdF32: + result = select_face( + select_f32(x_neg, IR::F32(IR::Value::F32(1.0f)), IR::F32(IR::Value::F32(0.0f))), + select_f32(y_neg, IR::F32(IR::Value::F32(3.0f)), IR::F32(IR::Value::F32(2.0f))), + select_f32(z_neg, IR::F32(IR::Value::F32(5.0f)), IR::F32(IR::Value::F32(4.0f)))); + break; + case IR::Opcode::CubeScF32: + result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x)); + break; + case IR::Opcode::CubeTcF32: result = select_face(ny, select_f32(y_neg, nz, z), ny); break; + case IR::Opcode::CubeMaF32: { + const auto two = IR::F32(IR::Value::F32(2.0f)); + result = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})), + IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})), + IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {z, two}))); + break; + } + default: return false; + } + WriteOperand(inst.dst, result); + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Integer.cpp b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp new file mode 100644 index 000000000..d0d9a25fe --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Integer.cpp @@ -0,0 +1,630 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +bool Translator::TranslateInteger16Operation(const IR::Instruction& inst) { + const auto mask16 = [&](IR::U32 value) { + return ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))); + }; + IR::Value result; + switch (inst.op) { + case IR::Opcode::ShiftLeftLogicalU16: + case IR::Opcode::ShiftRightLogicalU16: + case IR::Opcode::ShiftRightArithmeticI16: { + const bool arithmetic = inst.op == IR::Opcode::ShiftRightArithmeticI16; + const auto value = ReadU16AsU32(inst.src[0], arithmetic); + const auto count = + ir.BitwiseAnd(ReadU16AsU32(inst.src[1], false), IR::U32(IR::Value(15u))); + if (inst.op == IR::Opcode::ShiftLeftLogicalU16) { + result = ir.ShiftLeftLogical(value, count); + } else if (arithmetic) { + result = ir.ShiftRightArithmetic(value, count); + } else { + result = ir.ShiftRightLogical(value, count); + } + break; + } + case IR::Opcode::IAddU16: + result = ir.IAdd(ReadU16AsU32(inst.src[0], false), ReadU16AsU32(inst.src[1], false)); + break; + case IR::Opcode::ISubI16: + result = ir.ISub(ReadU16AsU32(inst.src[0], false), ReadU16AsU32(inst.src[1], false)); + break; + case IR::Opcode::IMinI16: + case IR::Opcode::IMaxI16: + case IR::Opcode::UMinU16: + case IR::Opcode::UMaxU16: { + const bool sign = inst.op == IR::Opcode::IMinI16 || inst.op == IR::Opcode::IMaxI16; + IR::ValueOpcode opcode; + switch (inst.op) { + case IR::Opcode::IMinI16: opcode = IR::ValueOpcode::SMin32; break; + case IR::Opcode::IMaxI16: opcode = IR::ValueOpcode::SMax32; break; + case IR::Opcode::UMinU16: opcode = IR::ValueOpcode::UMin32; break; + default: opcode = IR::ValueOpcode::UMax32; break; + } + result = + ir.Emit(opcode, {ReadU16AsU32(inst.src[0], sign), ReadU16AsU32(inst.src[1], sign)}); + break; + } + default: return false; + } + WriteU16(inst.dst, mask16(IR::U32(result))); + return true; +} + +bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) { + const auto lane = [&](const IR::Operand& operand, bool high, bool sign) { + return ReadU16LaneAsU32(operand, high, sign); + }; + const auto mask_count = [&](IR::U32 value) { + return ir.BitwiseAnd(value, IR::U32(IR::Value(15u))); + }; + const auto translate_lane = [&](bool high) -> IR::U32 { + if (inst.op == IR::Opcode::PackedLshlrevB16 || inst.op == IR::Opcode::PackedLshrrevB16 || + inst.op == IR::Opcode::PackedAshrrevI16) { + const auto count = mask_count(lane(inst.src[0], high, false)); + const bool arithmetic = inst.op == IR::Opcode::PackedAshrrevI16; + const auto value = lane(inst.src[1], high, arithmetic); + if (inst.op == IR::Opcode::PackedLshlrevB16) { + return ir.ShiftLeftLogical(value, count); + } + return arithmetic ? ir.ShiftRightArithmetic(value, count) + : ir.ShiftRightLogical(value, count); + } + + const bool signed_minmax = + inst.op == IR::Opcode::PackedMaxI16 || inst.op == IR::Opcode::PackedMinI16; + const auto lhs = lane(inst.src[0], high, signed_minmax); + const auto rhs = lane(inst.src[1], high, signed_minmax); + switch (inst.op) { + case IR::Opcode::PackedMadI16: + case IR::Opcode::PackedMadU16: + return ir.IAdd(ir.IMul(lhs, rhs), + lane(inst.src[2], high, inst.op == IR::Opcode::PackedMadI16)); + case IR::Opcode::PackedMulLoU16: return ir.IMul(lhs, rhs); + case IR::Opcode::PackedAddI16: + case IR::Opcode::PackedAddU16: return ir.IAdd(lhs, rhs); + case IR::Opcode::PackedSubI16: + case IR::Opcode::PackedSubU16: return ir.ISub(lhs, rhs); + case IR::Opcode::PackedMaxI16: + return IR::U32(ir.Emit(IR::ValueOpcode::SMax32, {lhs, rhs})); + case IR::Opcode::PackedMinI16: + return IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {lhs, rhs})); + case IR::Opcode::PackedMaxU16: + return IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); + case IR::Opcode::PackedMinU16: + return IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); + default: EXIT("invalid packed integer opcode"); + } + }; + + switch (inst.op) { + case IR::Opcode::PackedMadI16: + case IR::Opcode::PackedMulLoU16: + case IR::Opcode::PackedAddI16: + case IR::Opcode::PackedSubI16: + case IR::Opcode::PackedLshlrevB16: + case IR::Opcode::PackedLshrrevB16: + case IR::Opcode::PackedAshrrevI16: + case IR::Opcode::PackedMaxI16: + case IR::Opcode::PackedMinI16: + case IR::Opcode::PackedMadU16: + case IR::Opcode::PackedAddU16: + case IR::Opcode::PackedSubU16: + case IR::Opcode::PackedMaxU16: + case IR::Opcode::PackedMinU16: + WriteOperand(inst.dst, PackU16Lanes(translate_lane(false), translate_lane(true))); + return true; + default: return false; + } +} + +bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) { + if (!current_per_invocation_masks) { + return false; + } + + const auto lhs = [&] { return ReadMask(inst.src[0]); }; + const auto rhs = [&] { return ReadMask(inst.src[1]); }; + IR::U1 result; + switch (inst.op) { + case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break; + case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break; + case IR::Opcode::BitwiseXorU64: + result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})); + break; + case IR::Opcode::BitwiseAndNotU64: + result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs())); + break; + case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break; + case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break; + case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break; + case IR::Opcode::BitwiseXnorU64: + result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}))); + break; + case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break; + default: return false; + } + WriteMask64(inst.dst, result); + return true; +} + +bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) { + IR::ValueOpcode opcode {}; + IR::Type type = IR::Type::U32; + switch (inst.op) { + case IR::Opcode::AbsI32: opcode = IR::ValueOpcode::IAbs32; break; + case IR::Opcode::IAddU32: opcode = IR::ValueOpcode::IAdd32; break; + case IR::Opcode::ISubU32: opcode = IR::ValueOpcode::ISub32; break; + case IR::Opcode::IMulU32: opcode = IR::ValueOpcode::IMul32; break; + case IR::Opcode::UMulHighU32: opcode = IR::ValueOpcode::UMulHi; break; + case IR::Opcode::SMulHighI32: opcode = IR::ValueOpcode::SMulHi; break; + case IR::Opcode::IMinI32: opcode = IR::ValueOpcode::SMin32; break; + case IR::Opcode::IMaxI32: opcode = IR::ValueOpcode::SMax32; break; + case IR::Opcode::UMinU32: opcode = IR::ValueOpcode::UMin32; break; + case IR::Opcode::UMaxU32: opcode = IR::ValueOpcode::UMax32; break; + case IR::Opcode::IMin3I32: opcode = IR::ValueOpcode::SMinTri32; break; + case IR::Opcode::IMax3I32: opcode = IR::ValueOpcode::SMaxTri32; break; + case IR::Opcode::IMed3I32: opcode = IR::ValueOpcode::SMedTri32; break; + case IR::Opcode::UMin3U32: opcode = IR::ValueOpcode::UMinTri32; break; + case IR::Opcode::UMax3U32: opcode = IR::ValueOpcode::UMaxTri32; break; + case IR::Opcode::UMed3U32: opcode = IR::ValueOpcode::UMedTri32; break; + case IR::Opcode::BitwiseAndU32: opcode = IR::ValueOpcode::BitwiseAnd32; break; + case IR::Opcode::BitwiseOrU32: opcode = IR::ValueOpcode::BitwiseOr32; break; + case IR::Opcode::BitwiseXorU32: opcode = IR::ValueOpcode::BitwiseXor32; break; + case IR::Opcode::BitwiseNotU32: opcode = IR::ValueOpcode::BitwiseNot32; break; + case IR::Opcode::BitReverseU32: opcode = IR::ValueOpcode::BitReverse32; break; + case IR::Opcode::BitCountU32: opcode = IR::ValueOpcode::BitCount32; break; + case IR::Opcode::BitCountU64: + opcode = IR::ValueOpcode::BitCount64; + type = IR::Type::U64; + break; + case IR::Opcode::FindLsbU32: opcode = IR::ValueOpcode::FindILsb32; break; + case IR::Opcode::BitwiseAndU64: + opcode = IR::ValueOpcode::BitwiseAnd64; + type = IR::Type::U64; + break; + case IR::Opcode::BitwiseOrU64: + opcode = IR::ValueOpcode::BitwiseOr64; + type = IR::Type::U64; + break; + case IR::Opcode::ShiftLeftLogicalU32: opcode = IR::ValueOpcode::ShiftLeftLogical32; break; + case IR::Opcode::ShiftRightLogicalU32: opcode = IR::ValueOpcode::ShiftRightLogical32; break; + case IR::Opcode::ShiftRightArithmeticI32: + opcode = IR::ValueOpcode::ShiftRightArithmetic32; + break; + case IR::Opcode::ShiftLeftLogicalU64: + opcode = IR::ValueOpcode::ShiftLeftLogical64; + type = IR::Type::U64; + break; + case IR::Opcode::ShiftRightLogicalU64: + opcode = IR::ValueOpcode::ShiftRightLogical64; + type = IR::Type::U64; + break; + default: return false; + } + + std::array args; + for (uint32_t index = 0; index < inst.src_count; index++) { + const auto arg_type = IR::ArgTypeOf(opcode, index); + args[index] = ReadOperand(inst.src[index], arg_type == IR::Type::Void ? type : arg_type); + if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical32 || + opcode == IR::ValueOpcode::ShiftRightLogical32 || + opcode == IR::ValueOpcode::ShiftRightArithmetic32)) { + args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(31u))); + } else if (index == 1u && (opcode == IR::ValueOpcode::ShiftLeftLogical64 || + opcode == IR::ValueOpcode::ShiftRightLogical64 || + opcode == IR::ValueOpcode::ShiftRightArithmetic64)) { + args[index] = ir.BitwiseAnd(IR::U32(args[index]), IR::U32(IR::Value(63u))); + } + } + IR::Value result; + switch (inst.src_count) { + case 1: result = ir.Emit(opcode, {args[0]}); break; + case 2: result = ir.Emit(opcode, {args[0], args[1]}); break; + case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break; + default: EXIT("invalid simple integer source count: %u", inst.src_count); + } + WriteOperand(inst.dst, result); + if (type == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + IR::U1 mask; + if (inst.op == IR::Opcode::BitwiseAndU64) { + mask = ir.LogicalAnd(ReadMask(inst.src[0]), ReadMask(inst.src[1])); + } else if (inst.op == IR::Opcode::BitwiseOrU64) { + mask = ir.LogicalOr(ReadMask(inst.src[0]), ReadMask(inst.src[1])); + } + if (!mask.IsEmpty()) { + ir.SetThreadBitScalarReg(static_cast(inst.dst.reg.index), mask); + } + } + return true; +} + +bool Translator::TranslateComposedInteger(const IR::Instruction& inst) { + const auto binary_u32 = [&](auto operation) { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + return operation(lhs, rhs); + }; + const auto binary_u64 = [&](auto operation) { + const auto lhs = UnpackU64(ReadU64(inst.src[0])); + const auto rhs = UnpackU64(ReadU64(inst.src[1])); + return ir.PackUint2x32(operation(lhs[0], rhs[0]), operation(lhs[1], rhs[1])); + }; + + IR::Value result; + switch (inst.op) { + case IR::Opcode::BitwiseAndNotU32: + result = binary_u32( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); }); + break; + case IR::Opcode::BitwiseOrNotU32: + result = binary_u32( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); }); + break; + case IR::Opcode::BitwiseNandU32: + result = binary_u32( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); }); + break; + case IR::Opcode::BitwiseNorU32: + result = binary_u32( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); }); + break; + case IR::Opcode::BitwiseXnorU32: + result = binary_u32( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); }); + break; + case IR::Opcode::BitwiseXorU64: + result = binary_u64([&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseXor(lhs, rhs); }); + break; + case IR::Opcode::BitwiseAndNotU64: + result = binary_u64( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseAnd(lhs, ir.BitwiseNot(rhs)); }); + break; + case IR::Opcode::BitwiseOrNotU64: + result = binary_u64( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseOr(lhs, ir.BitwiseNot(rhs)); }); + break; + case IR::Opcode::BitwiseNandU64: + result = binary_u64( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseAnd(lhs, rhs)); }); + break; + case IR::Opcode::BitwiseNorU64: + result = binary_u64( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseOr(lhs, rhs)); }); + break; + case IR::Opcode::BitwiseXnorU64: + result = binary_u64( + [&](IR::U32 lhs, IR::U32 rhs) { return ir.BitwiseNot(ir.BitwiseXor(lhs, rhs)); }); + break; + case IR::Opcode::BitwiseNotU64: { + const auto source = UnpackU64(ReadU64(inst.src[0])); + result = ir.PackUint2x32(ir.BitwiseNot(source[0]), ir.BitwiseNot(source[1])); + break; + } + case IR::Opcode::BitwiseAndOrU32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const auto add = ReadU32(inst.src[2]); + result = ir.BitwiseOr(ir.BitwiseAnd(lhs, rhs), add); + break; + } + case IR::Opcode::BitwiseOr3U32: + result = ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src[0]), ReadU32(inst.src[1])), + ReadU32(inst.src[2])); + break; + case IR::Opcode::BitwiseXor3U32: + result = ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src[0]), ReadU32(inst.src[1])), + ReadU32(inst.src[2])); + break; + case IR::Opcode::FindMsbFromHighU32: { + const auto source = ReadU32(inst.src[0]); + const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {source})); + const auto position = ir.ISub(IR::U32(IR::Value(31u)), msb); + result = ir.Select(ir.INotEqual(source, IR::U32(IR::Value(0u))), position, + IR::U32(IR::Value(0xffffffffu))); + break; + } + case IR::Opcode::FindMsbFromHighU64: { + const auto source = ReadU64(inst.src[0]); + const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source})); + const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb); + const auto nonzero = IR::U1( + ir.Emit(IR::ValueOpcode::INotEqual64, + {source, ir.PackUint2x32(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))})); + result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu))); + break; + } + default: return false; + } + WriteOperand(inst.dst, result); + if (result.GetType() == IR::Type::U64 && inst.dst.kind == IR::OperandKind::Register && + inst.dst.reg.file == IR::RegisterFile::Scalar) { + const auto lhs = ReadMask(inst.src[0]); + IR::U1 mask; + switch (inst.op) { + case IR::Opcode::BitwiseXorU64: + mask = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])})); + break; + case IR::Opcode::BitwiseAndNotU64: + mask = ir.LogicalAnd(lhs, ir.LogicalNot(ReadMask(inst.src[1]))); + break; + case IR::Opcode::BitwiseOrNotU64: + mask = ir.LogicalOr(lhs, ir.LogicalNot(ReadMask(inst.src[1]))); + break; + case IR::Opcode::BitwiseNandU64: + mask = ir.LogicalNot(ir.LogicalAnd(lhs, ReadMask(inst.src[1]))); + break; + case IR::Opcode::BitwiseNorU64: + mask = ir.LogicalNot(ir.LogicalOr(lhs, ReadMask(inst.src[1]))); + break; + case IR::Opcode::BitwiseXnorU64: + mask = ir.LogicalNot( + IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs, ReadMask(inst.src[1])}))); + break; + case IR::Opcode::BitwiseNotU64: mask = ir.LogicalNot(lhs); break; + default: break; + } + if (!mask.IsEmpty()) { + ir.SetThreadBitScalarReg(static_cast(inst.dst.reg.index), mask); + } + } + return true; +} + +bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) { + const auto imm = [](uint32_t value) { return IR::U32(IR::Value(value)); }; + const auto mask = [&](IR::U32 value, uint32_t bits) { return ir.BitwiseAnd(value, imm(bits)); }; + const auto extract = [&](IR::U32 value, IR::U32 offset, IR::U32 width, bool sign) { + return IR::U32( + ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract, + {value, offset, width})); + }; + const auto right_mask32 = [&](IR::U32 count) { + return IR::U32( + ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), imm(0), count})); + }; + const auto select_u64 = [&](IR::U1 condition, IR::U64 true_value, IR::U64 false_value) { + const auto lhs = UnpackU64(true_value); + const auto rhs = UnpackU64(false_value); + return ir.PackUint2x32(ir.Select(condition, lhs[0], rhs[0]), + ir.Select(condition, lhs[1], rhs[1])); + }; + const auto right_mask64 = [&](IR::U32 count) { + const auto below32 = IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {count, imm(32)})); + const auto above32 = IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThan32, {count, imm(32)})); + const auto low_count = ir.Select(below32, count, imm(32)); + const auto high_count = ir.Select(above32, ir.ISub(count, imm(32)), imm(0)); + return ir.PackUint2x32(right_mask32(low_count), right_mask32(high_count)); + }; + + IR::Value result; + switch (inst.op) { + case IR::Opcode::IMadI24U32: + case IR::Opcode::UMadU24U32: + case IR::Opcode::IMulI24U32: + case IR::Opcode::UMulU24U32: { + const bool sign = + inst.op == IR::Opcode::IMadI24U32 || inst.op == IR::Opcode::IMulI24U32; + const auto lhs = extract(ReadU32(inst.src[0]), imm(0), imm(24), sign); + const auto rhs = extract(ReadU32(inst.src[1]), imm(0), imm(24), sign); + auto value = ir.IMul(lhs, rhs); + if (inst.src_count == 3) { + value = ir.IAdd(value, ReadU32(inst.src[2])); + } + result = value; + break; + } + case IR::Opcode::UMadU64U32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const auto add = UnpackU64(ReadU64(inst.src[2])); + const auto mul_low = ir.IMul(lhs, rhs); + const auto mul_high = IR::U32(ir.Emit(IR::ValueOpcode::UMulHi, {lhs, rhs})); + const auto low = ir.IAdd(mul_low, add[0]); + const auto carry_low = ir.ULessThan(low, mul_low); + const auto high0 = ir.IAdd(mul_high, add[1]); + const auto carry0 = ir.ULessThan(high0, mul_high); + const auto high = ir.IAdd(high0, ir.Select(carry_low, imm(1), imm(0))); + const auto carry1 = ir.ULessThan(high, high0); + WriteOperand(inst.dst, ir.PackUint2x32(low, high)); + if (inst.dst2.kind != IR::OperandKind::Null) { + WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1)); + } + return true; + } + case IR::Opcode::SadU32: { + const auto lhs = ReadU32(inst.src[0]); + const auto rhs = ReadU32(inst.src[1]); + const auto lo = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs})); + const auto hi = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs})); + result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src[2])); + break; + } + case IR::Opcode::IAdd3U32: + result = + ir.IAdd(ir.IAdd(ReadU32(inst.src[0]), ReadU32(inst.src[1])), ReadU32(inst.src[2])); + break; + case IR::Opcode::BitClearU32: + case IR::Opcode::BitSetU32: { + const auto bit = ir.ShiftLeftLogical(imm(1), mask(ReadU32(inst.src[1]), 31u)); + result = inst.op == IR::Opcode::BitClearU32 + ? IR::Value(ir.BitwiseAnd(ReadU32(inst.src[0]), ir.BitwiseNot(bit))) + : IR::Value(ir.BitwiseOr(ReadU32(inst.src[0]), bit)); + break; + } + case IR::Opcode::BitCountAddU32: + result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src[0])})), + ReadU32(inst.src[1])); + break; + case IR::Opcode::MaskedBitCountLowU32: + case IR::Opcode::MaskedBitCountHighU32: { + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto local = mask(lane, 31u); + const auto below = ir.ISub(ir.ShiftLeftLogical(imm(1), local), imm(1)); + const auto high_lane = + IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, imm(32)})); + const auto thread_mask = inst.op == IR::Opcode::MaskedBitCountLowU32 + ? ir.Select(high_lane, imm(0xffffffffu), below) + : ir.Select(high_lane, below, imm(0)); + const auto active = ir.BitwiseAnd(ReadU32(inst.src[0]), thread_mask); + result = ir.IAdd(IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active})), + ReadU32(inst.src[1])); + break; + } + case IR::Opcode::BitReplicateB64B32: { + const auto replicate = [&](IR::U32 value) { + auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, imm(8))); + bits = ir.BitwiseAnd(bits, imm(0x00ff00ffu)); + bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(4))); + bits = ir.BitwiseAnd(bits, imm(0x0f0f0f0fu)); + bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(2))); + bits = ir.BitwiseAnd(bits, imm(0x33333333u)); + bits = ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1))); + bits = ir.BitwiseAnd(bits, imm(0x55555555u)); + return ir.BitwiseOr(bits, ir.ShiftLeftLogical(bits, imm(1))); + }; + const auto source = ReadU32(inst.src[0]); + result = ir.PackUint2x32(replicate(mask(source, 0xffffu)), + replicate(ir.ShiftRightLogical(source, imm(16)))); + break; + } + case IR::Opcode::BitFieldMaskU32: { + const auto count = mask(ReadU32(inst.src[0]), 31u); + const auto offset = mask(ReadU32(inst.src[1]), 31u); + result = + ir.Emit(IR::ValueOpcode::BitFieldInsert, {imm(0), imm(0xffffffffu), offset, count}); + break; + } + case IR::Opcode::BitFieldMaskU64: { + const auto count = mask(ReadU32(inst.src[0]), 63u); + const auto offset = mask(ReadU32(inst.src[1]), 63u); + result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {right_mask64(count), offset}); + break; + } + case IR::Opcode::BitFieldExtractU32: { + const auto source = ReadU32(inst.src[0]); + const auto field = ReadU32(inst.src[1]); + const auto offset = extract(field, imm(0), imm(5), false); + const auto count = + IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {extract(field, imm(16), imm(7), false), + ir.ISub(imm(32), offset)})); + result = ir.Emit(IR::ValueOpcode::BitFieldUExtract, {source, offset, count}); + break; + } + case IR::Opcode::BitFieldExtractU64: { + const auto source = ReadU64(inst.src[0]); + const auto field = ReadU32(inst.src[1]); + const auto offset = extract(field, imm(0), imm(6), false); + const auto raw_count = extract(field, imm(16), imm(7), false); + const auto available = ir.ISub(imm(64), offset); + const auto count = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {raw_count, available})); + const auto shifted = + IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {source, offset})); + result = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, right_mask64(count)}); + break; + } + case IR::Opcode::BitFieldExtract3U32: + case IR::Opcode::BitFieldExtract3I32: { + const auto source = ReadU32(inst.src[0]); + const auto offset = mask(ReadU32(inst.src[1]), 31u); + const auto count = + IR::U32(ir.Emit(IR::ValueOpcode::UMin32, + {mask(ReadU32(inst.src[2]), 31u), ir.ISub(imm(32), offset)})); + if (inst.op == IR::Opcode::BitFieldExtract3I32) { + result = + ir.BitwiseAnd(ir.ShiftRightArithmetic(source, offset), right_mask32(count)); + } else { + result = ir.Emit(IR::ValueOpcode::BitFieldUExtract, {source, offset, count}); + } + break; + } + case IR::Opcode::BitFieldInsertSelectU32: { + const auto bits = ReadU32(inst.src[0]); + const auto insert = ReadU32(inst.src[1]); + const auto base = ReadU32(inst.src[2]); + result = + ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base)); + break; + } + case IR::Opcode::BitCompare0B32: + case IR::Opcode::BitCompare1B32: { + const auto value = ReadU32(inst.src[0]); + const auto bit = extract(value, mask(ReadU32(inst.src[1]), 31u), imm(1), false); + const auto expected = imm(inst.op == IR::Opcode::BitCompare1B32 ? 1u : 0u); + WriteOperand(inst.dst, ir.IEqual(bit, expected)); + return true; + } + case IR::Opcode::AlignBitU32: { + const auto hi = ReadU32(inst.src[0]); + const auto lo = ReadU32(inst.src[1]); + const auto shift = mask(ReadU32(inst.src[2]), 31u); + const auto lo_part = ir.ShiftRightLogical(lo, shift); + const auto hi_part_raw = ir.ShiftLeftLogical(hi, mask(ir.ISub(imm(32), shift), 31u)); + const auto hi_part = ir.Select(ir.INotEqual(shift, imm(0)), hi_part_raw, imm(0)); + result = ir.BitwiseOr(lo_part, hi_part); + break; + } + case IR::Opcode::ShiftLeftAddU32: + result = + ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src[0]), mask(ReadU32(inst.src[1]), 31u)), + ReadU32(inst.src[2])); + break; + case IR::Opcode::AddShiftLeftU32: + result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src[0]), ReadU32(inst.src[1])), + mask(ReadU32(inst.src[2]), 31u)); + break; + case IR::Opcode::XorAddU32: + result = ir.IAdd(ir.BitwiseXor(ReadU32(inst.src[0]), ReadU32(inst.src[1])), + ReadU32(inst.src[2])); + break; + case IR::Opcode::ShiftLeftOrU32: + result = ir.BitwiseOr( + ir.ShiftLeftLogical(ReadU32(inst.src[0]), mask(ReadU32(inst.src[1]), 31u)), + ReadU32(inst.src[2])); + break; + case IR::Opcode::SelectU32: + result = + ir.Select(ReadCondition(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2])); + break; + case IR::Opcode::SelectMaskU32: + result = ir.Select(ReadMask(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2])); + break; + case IR::Opcode::SelectF32Bits: + case IR::Opcode::SelectMaskF32Bits: { + const auto condition = inst.op == IR::Opcode::SelectMaskF32Bits + ? ReadMask(inst.src[0]) + : ReadCondition(inst.src[0]); + result = ir.Emit(IR::ValueOpcode::SelectF32, + {condition, ReadOperand(inst.src[1], IR::Type::F32), + ReadOperand(inst.src[2], IR::Type::F32)}); + break; + } + case IR::Opcode::SelectU64: + result = + select_u64(ReadCondition(inst.src[0]), ReadU64(inst.src[1]), ReadU64(inst.src[2])); + break; + case IR::Opcode::PackLowLowU16: + case IR::Opcode::PackLowHighU16: + case IR::Opcode::PackHighHighU16: + case IR::Opcode::PackU16U32: { + const bool high0 = inst.op == IR::Opcode::PackHighHighU16; + const bool high1 = inst.op == IR::Opcode::PackLowHighU16 || high0; + const auto lo = + high0 ? ir.ShiftRightLogical(ReadU32(inst.src[0]), imm(16)) : ReadU32(inst.src[0]); + const auto hi = + high1 ? ir.ShiftRightLogical(ReadU32(inst.src[1]), imm(16)) : ReadU32(inst.src[1]); + result = + ir.BitwiseOr(mask(lo, 0xffffu), ir.ShiftLeftLogical(mask(hi, 0xffffu), imm(16))); + break; + } + default: return false; + } + WriteOperand(inst.dst, result); + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp new file mode 100644 index 000000000..fa8d318cb --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -0,0 +1,648 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +IR::MemoryFlags Translator::AddMemoryInfo(const IR::Instruction& inst) { + const auto index = static_cast(value_program.memory_info.size()); + value_program.memory_info.push_back(inst.memory); + return {.index = index, .pc = inst.pc}; +} + +IR::U32 Translator::GetResourceDword(uint32_t index, uint32_t dword) { + return ReadScalarCode(index * 4u + dword); +} + +IR::Value Translator::GetBufferResource(const IR::MemoryInfo& memory) { + return ir.Emit(IR::ValueOpcode::GetBufferResource, + {GetResourceDword(memory.resource, 0), GetResourceDword(memory.resource, 1), + GetResourceDword(memory.resource, 2), GetResourceDword(memory.resource, 3)}); +} + +IR::Value Translator::GetAddressResource(IR::Value low, IR::Value high) { + return ir.Emit(IR::ValueOpcode::GetAddressResource, {low, high}); +} + +Translator::AddressOperands Translator::ReadAddressOperands(const IR::Instruction& inst, + uint32_t first_source) { + const auto low = ReadU32(inst.src[first_source]); + const auto& high_or_base = inst.src[first_source + 1u]; + if (inst.memory.kind == IR::ResourceKind::Scratch) { + const auto offset = high_or_base.kind == IR::OperandKind::Register && + high_or_base.reg.file != IR::RegisterFile::Vector + ? ReadU32(high_or_base) + : low; + return {GetAddressResource(offset, IR::Value(0u)), offset, IR::Value(0u)}; + } + if (inst.memory.kind == IR::ResourceKind::Global && + high_or_base.kind == IR::OperandKind::Register && + high_or_base.reg.file != IR::RegisterFile::Vector) { + const auto base_low = ReadU32(high_or_base); + const auto base_high = ReadU32(OffsetOperand(high_or_base, 1u)); + return {GetAddressResource(base_low, base_high), low, IR::Value(0u)}; + } + const auto high = ReadU32(high_or_base); + return {GetAddressResource(low, high), low, high}; +} + +IR::Value Translator::GetScalarAddressResource(uint32_t base) { + return GetAddressResource(ReadScalarCode(base), ReadScalarCode(base + 1u)); +} + +IR::Value Translator::GetImageResource(const IR::MemoryInfo& memory) { + return ir.Emit(IR::ValueOpcode::GetImageResource, + {GetResourceDword(memory.resource, 0), GetResourceDword(memory.resource, 1), + GetResourceDword(memory.resource, 2), GetResourceDword(memory.resource, 3), + GetResourceDword(memory.resource, 4), GetResourceDword(memory.resource, 5), + GetResourceDword(memory.resource, 6), GetResourceDword(memory.resource, 7)}); +} + +IR::Value Translator::GetSamplerResource(const IR::MemoryInfo& memory) { + return ir.Emit(IR::ValueOpcode::GetSamplerResource, + {GetResourceDword(memory.sampler, 0), GetResourceDword(memory.sampler, 1), + GetResourceDword(memory.sampler, 2), GetResourceDword(memory.sampler, 3)}); +} + +IR::Value Translator::GetSharedResource(IR::ResourceKind kind) { + return ir.Emit(kind == IR::ResourceKind::Gds ? IR::ValueOpcode::GetGdsResource + : IR::ValueOpcode::GetLdsResource); +} + +IR::Value Translator::MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base) { + std::array components {}; + components[0] = ReadRawU32(PlainOperand(base)); + const auto nsa_components = + std::min(inst.memory.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents); + for (uint32_t index = 1; index < components.size(); index++) { + if (index - 1u < nsa_components) { + components[index] = + ir.GetVectorReg(static_cast(inst.memory.image_nsa_addr[index - 1u])); + } else { + components[index] = ReadRawU32(OffsetOperand(PlainOperand(base), index)); + } + } + return ir.Emit(IR::ValueOpcode::MakeImageAddress, + {components[0], components[1], components[2], components[3], components[4], + components[5], components[6], components[7], components[8], components[9], + components[10], components[11], components[12]}); +} + +IR::Value Translator::ConstructU32x4(const IR::Operand& base, uint32_t count) { + std::array components {IR::Value(0u), IR::Value(0u), IR::Value(0u), + IR::Value(0u)}; + for (uint32_t index = 0; index < std::min(count, 4u); index++) { + components[index] = ReadRawU32(OffsetOperand(PlainOperand(base), index)); + } + return ir.Emit(IR::ValueOpcode::CompositeConstructU32x4, + {components[0], components[1], components[2], components[3]}); +} + +void Translator::WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask, + uint32_t component_limit) { + const auto mask = dmask != 0u ? dmask : 1u; + uint32_t dst_index = 0; + for (uint32_t component = 0; component < component_limit; component++) { + if (((mask >> component) & 1u) == 0u) { + continue; + } + WriteOperand( + OffsetOperand(dst, dst_index++), + ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {value, IR::Value(component)})); + } +} + +IR::ValueOpcode Translator::ImageAtomicOpcode(IR::Opcode opcode) { + switch (opcode) { + case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::ImageAtomicIAdd32; + case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::ImageAtomicUMin32; + case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::ImageAtomicUMax32; + case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::ImageAtomicAnd32; + case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::ImageAtomicOr32; + case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::ImageAtomicXor32; + default: EXIT("invalid image atomic opcode"); + } +} + +BufferAddressValues Translator::ReadBufferAddress(const IR::Instruction& inst, + uint32_t first_source) { + uint32_t cursor = first_source; + const auto next = [&]() { + return cursor < inst.src_count ? ReadU32(inst.src[cursor++]) : IR::U32(IR::Value(0u)); + }; + const auto index = inst.memory.idxen ? next() : IR::U32(IR::Value(0u)); + const auto offset = inst.memory.offen ? next() : IR::U32(IR::Value(0u)); + const auto soffset = next(); + return {index, offset, soffset}; +} + +IR::U32 Translator::WidenSubdword(IR::Value value, uint32_t bits, bool sign) { + IR::U32 widened = bits == 8u ? IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U8, {value})) + : IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {value})); + if (sign) { + widened = IR::U32( + ir.Emit(IR::ValueOpcode::BitFieldSExtract, {widened, IR::Value(0u), IR::Value(bits)})); + } + return widened; +} + +IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) { + return bits == 8u ? ir.Emit(IR::ValueOpcode::ConvertU8U32, {value}) + : ir.Emit(IR::ValueOpcode::ConvertU16U32, {value}); +} + +IR::ValueOpcode Translator::BufferAtomicOpcode(IR::Opcode opcode) { + switch (opcode) { + case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::BufferAtomicSwap32; + case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::BufferAtomicIAdd32; + case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::BufferAtomicISub32; + case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::BufferAtomicSMin32; + case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::BufferAtomicUMin32; + case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::BufferAtomicSMax32; + case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::BufferAtomicUMax32; + case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::BufferAtomicAnd32; + case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::BufferAtomicOr32; + case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::BufferAtomicXor32; + case IR::Opcode::AtomicFMinF32: return IR::ValueOpcode::BufferAtomicFMin32; + case IR::Opcode::AtomicFMaxF32: return IR::ValueOpcode::BufferAtomicFMax32; + default: EXIT("invalid buffer atomic opcode"); + } +} + +IR::ValueOpcode Translator::SharedAtomicOpcode(IR::Opcode opcode, bool gds) { + if (gds) { + switch (opcode) { + case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::GdsAtomicSwap32; + case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::GdsAtomicIAdd32; + case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::GdsAtomicISub32; + case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::GdsAtomicSMin32; + case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::GdsAtomicUMin32; + case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::GdsAtomicSMax32; + case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::GdsAtomicUMax32; + case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::GdsAtomicAnd32; + case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::GdsAtomicOr32; + case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::GdsAtomicXor32; + default: EXIT("invalid GDS atomic opcode"); + } + } + switch (opcode) { + case IR::Opcode::AtomicSwapU32: return IR::ValueOpcode::SharedAtomicSwap32; + case IR::Opcode::AtomicAddU32: return IR::ValueOpcode::SharedAtomicIAdd32; + case IR::Opcode::AtomicSubU32: return IR::ValueOpcode::SharedAtomicISub32; + case IR::Opcode::AtomicSMinI32: return IR::ValueOpcode::SharedAtomicSMin32; + case IR::Opcode::AtomicUMinU32: return IR::ValueOpcode::SharedAtomicUMin32; + case IR::Opcode::AtomicSMaxI32: return IR::ValueOpcode::SharedAtomicSMax32; + case IR::Opcode::AtomicUMaxU32: return IR::ValueOpcode::SharedAtomicUMax32; + case IR::Opcode::AtomicAndU32: return IR::ValueOpcode::SharedAtomicAnd32; + case IR::Opcode::AtomicOrU32: return IR::ValueOpcode::SharedAtomicOr32; + case IR::Opcode::AtomicXorU32: return IR::ValueOpcode::SharedAtomicXor32; + default: EXIT("invalid LDS atomic opcode"); + } +} + +bool Translator::IsBufferLoadOperation(IR::Opcode opcode) { + return opcode == IR::Opcode::BufferLoadUbyte || opcode == IR::Opcode::BufferLoadSbyte || + opcode == IR::Opcode::BufferLoadUshort || opcode == IR::Opcode::BufferLoadSshort || + opcode == IR::Opcode::BufferLoadDword; +} + +bool Translator::TranslateScalarMemory(const IR::Instruction& inst) { + switch (inst.op) { + case IR::Opcode::LoadSrtDword: { + const auto resource = ir.Emit(IR::ValueOpcode::GetSrtResource); + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::ReadConst, {resource, ReadU32(inst.src[0])})); + return true; + } + case IR::Opcode::SBufferLoadDword: { + const auto resource = GetBufferResource(inst.memory); + WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ReadConstBuffer, + {resource, ReadU32(inst.src[0])}, AddMemoryInfo(inst))); + return true; + } + case IR::Opcode::SLoadDword: { + const auto resource = GetScalarAddressResource(inst.memory.resource); + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::LoadAddressU32, + {resource, ReadU32(inst.src[0]), IR::Value(0u), IR::Value(true)}, + AddMemoryInfo(inst))); + return true; + } + default: return false; + } +} + +bool Translator::TranslateBufferLoad(const IR::Instruction& inst, + const BufferAddressValues* address_snapshot) { + IR::ValueOpcode opcode; + uint32_t bits; + bool sign; + switch (inst.op) { + case IR::Opcode::BufferLoadUbyte: + case IR::Opcode::BufferLoadSbyte: + opcode = IR::ValueOpcode::LoadBufferU8; + bits = 8u; + sign = inst.op == IR::Opcode::BufferLoadSbyte; + break; + case IR::Opcode::BufferLoadUshort: + case IR::Opcode::BufferLoadSshort: + opcode = IR::ValueOpcode::LoadBufferU16; + bits = 16u; + sign = inst.op == IR::Opcode::BufferLoadSshort; + break; + case IR::Opcode::BufferLoadDword: + opcode = IR::ValueOpcode::LoadBufferU32; + bits = 32u; + sign = false; + break; + default: return false; + } + const auto resource = GetBufferResource(inst.memory); + const auto address = + address_snapshot != nullptr ? *address_snapshot : ReadBufferAddress(inst, 0); + const auto loaded = + ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, ir.GetExec()}, + AddMemoryInfo(inst)); + WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); + return true; +} + +bool Translator::TranslateBufferStore(const IR::Instruction& inst) { + const auto resource = GetBufferResource(inst.memory); + const auto address = ReadBufferAddress(inst, 1); + const auto data = ReadU32(inst.src[0]); + IR::ValueOpcode opcode; + IR::Value value; + switch (inst.op) { + case IR::Opcode::BufferStoreByte: + opcode = IR::ValueOpcode::StoreBufferU8; + value = NarrowSubdword(data, 8u); + break; + case IR::Opcode::BufferStoreShort: + opcode = IR::ValueOpcode::StoreBufferU16; + value = NarrowSubdword(data, 16u); + break; + case IR::Opcode::BufferStoreDword: + opcode = IR::ValueOpcode::StoreBufferU32; + value = data; + break; + default: return false; + } + ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, value, ir.GetExec()}, + AddMemoryInfo(inst)); + return true; +} + +bool Translator::TranslateAtomicMemory(const IR::Instruction& inst) { + IR::Value result; + switch (inst.memory.kind) { + case IR::ResourceKind::Buffer: { + const auto resource = GetBufferResource(inst.memory); + const auto address = ReadBufferAddress(inst, 1); + result = ir.Emit(BufferAtomicOpcode(inst.op), + {resource, address.index, address.offset, address.soffset, + ReadU32(inst.src[0]), ir.GetExec()}, + AddMemoryInfo(inst)); + break; + } + case IR::ResourceKind::Image: + case IR::ResourceKind::ImageUint: + case IR::ResourceKind::StorageImage: + case IR::ResourceKind::StorageImageUint: { + const auto resource = GetImageResource(inst.memory); + const auto address = MakeImageAddress(inst, inst.src[1]); + const auto flags = AddMemoryInfo(inst); + result = ir.Emit(ImageAtomicOpcode(inst.op), + {resource, address, ReadU32(inst.src[0]), ir.GetExec()}, flags); + break; + } + case IR::ResourceKind::Lds: + case IR::ResourceKind::Gds: { + const bool gds = inst.memory.kind == IR::ResourceKind::Gds; + const auto resource = GetSharedResource(inst.memory.kind); + const auto address = + ir.IAdd(ReadU32(inst.src[1]), IR::U32(IR::Value(inst.memory.offset))); + result = ir.Emit(SharedAtomicOpcode(inst.op, gds), + {resource, address, ReadU32(inst.src[0]), ir.GetExec()}, + AddMemoryInfo(inst)); + break; + } + default: return false; + } + if (inst.dst.kind != IR::OperandKind::Null) { + WriteOperand(inst.dst, result); + } + return true; +} + +bool Translator::TranslateFlatLoad(const IR::Instruction& inst) { + IR::ValueOpcode opcode; + uint32_t bits; + bool sign; + switch (inst.op) { + case IR::Opcode::FlatLoadUbyte: + case IR::Opcode::FlatLoadSbyte: + opcode = IR::ValueOpcode::LoadAddressU8; + bits = 8u; + sign = inst.op == IR::Opcode::FlatLoadSbyte; + break; + case IR::Opcode::FlatLoadUshort: + case IR::Opcode::FlatLoadSshort: + opcode = IR::ValueOpcode::LoadAddressU16; + bits = 16u; + sign = inst.op == IR::Opcode::FlatLoadSshort; + break; + case IR::Opcode::FlatLoadDword: + opcode = IR::ValueOpcode::LoadAddressU32; + bits = 32u; + sign = false; + break; + default: return false; + } + const auto address = ReadAddressOperands(inst, 0); + const auto active = ir.GetExec(); + const auto count = bits == 32u ? std::min(inst.memory.data_dwords, 4u) : 1u; + for (uint32_t index = 0; index < count; index++) { + auto component = inst; + component.memory.offset += index * 4u; + component.memory.data_dwords = 1u; + component.memory.component_index = index; + const auto loaded = ir.Emit(opcode, {address.resource, address.low, address.high, active}, + AddMemoryInfo(component)); + WriteOperand(OffsetOperand(inst.dst, index), + bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); + } + return true; +} + +bool Translator::TranslateFlatStore(const IR::Instruction& inst) { + const auto data = ReadU32(inst.src[0]); + const auto address = ReadAddressOperands(inst, 1); + IR::ValueOpcode opcode; + IR::Value value; + switch (inst.op) { + case IR::Opcode::FlatStoreByte: + opcode = IR::ValueOpcode::StoreAddressU8; + value = NarrowSubdword(data, 8u); + break; + case IR::Opcode::FlatStoreShort: + opcode = IR::ValueOpcode::StoreAddressU16; + value = NarrowSubdword(data, 16u); + break; + case IR::Opcode::FlatStoreDword: + opcode = IR::ValueOpcode::StoreAddressU32; + value = data; + break; + default: return false; + } + ir.Emit(opcode, {address.resource, address.low, address.high, value, ir.GetExec()}, + AddMemoryInfo(inst)); + return true; +} + +bool Translator::TranslateImageMemory(const IR::Instruction& inst) { + const bool image = inst.memory.kind == IR::ResourceKind::Image || + inst.memory.kind == IR::ResourceKind::ImageUint || + inst.memory.kind == IR::ResourceKind::StorageImage || + inst.memory.kind == IR::ResourceKind::StorageImageUint; + if (!image) { + return false; + } + const auto resource = GetImageResource(inst.memory); + const auto address = + MakeImageAddress(inst, inst.op == IR::Opcode::ImageStore ? inst.src[1] : inst.src[0]); + const auto flags = AddMemoryInfo(inst); + switch (inst.op) { + case IR::Opcode::ImageGetResinfo: { + const auto result = + ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address}, flags); + WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u); + return true; + } + case IR::Opcode::ImageGetLod: { + const auto sampler = GetSamplerResource(inst.memory); + const auto result = + ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address}, flags); + WriteImageComponents(inst.dst, result, inst.memory.dmask, 2u); + return true; + } + case IR::Opcode::ImageLoad: { + const auto result = + ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()}, flags); + WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u); + return true; + } + case IR::Opcode::ImageStore: { + const auto data = ConstructU32x4(inst.src[0], inst.memory.data_dwords); + ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()}, flags); + return true; + } + case IR::Opcode::ImageSample: + case IR::Opcode::ImageGather4: { + const auto sampler = GetSamplerResource(inst.memory); + const auto opcode = inst.op == IR::Opcode::ImageSample + ? IR::ValueOpcode::ImageSampleRaw + : IR::ValueOpcode::ImageGatherRaw; + const auto result = ir.Emit(opcode, {resource, sampler, address}, flags); + const bool dref = + (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u; + if (inst.op == IR::Opcode::ImageSample && dref) { + const auto component = + ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(0u)}); + for (uint32_t index = 0; index < inst.memory.data_dwords; index++) { + WriteOperand(OffsetOperand(inst.dst, index), component); + } + } else if (inst.op == IR::Opcode::ImageGather4) { + for (uint32_t index = 0; index < inst.memory.data_dwords; index++) { + WriteOperand(OffsetOperand(inst.dst, index), + ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, + {result, IR::Value(index)})); + } + } else { + WriteImageComponents(inst.dst, result, inst.memory.dmask, 4u); + } + return true; + } + default: return false; + } +} + +bool Translator::TranslateSharedMemory(const IR::Instruction& inst) { + const bool shared = + inst.memory.kind == IR::ResourceKind::Lds || inst.memory.kind == IR::ResourceKind::Gds; + if (!shared && inst.op != IR::Opcode::DsSwizzleB32) { + return false; + } + const bool gds = inst.memory.kind == IR::ResourceKind::Gds; + const auto resource = shared ? GetSharedResource(inst.memory.kind) : IR::Value {}; + const auto shared_address = [&](uint32_t source) { + return ir.IAdd(ReadU32(inst.src[source]), IR::U32(IR::Value(inst.memory.offset))); + }; + switch (inst.op) { + case IR::Opcode::DsReadUbyte: + case IR::Opcode::DsReadSbyte: + case IR::Opcode::DsReadUshort: + case IR::Opcode::DsReadSshort: + case IR::Opcode::DsReadB32: { + IR::ValueOpcode opcode; + uint32_t bits; + bool sign; + if (inst.op == IR::Opcode::DsReadUbyte || inst.op == IR::Opcode::DsReadSbyte) { + opcode = gds ? IR::ValueOpcode::LoadGdsU8 : IR::ValueOpcode::LoadSharedU8; + bits = 8u; + sign = inst.op == IR::Opcode::DsReadSbyte; + } else if (inst.op == IR::Opcode::DsReadUshort || inst.op == IR::Opcode::DsReadSshort) { + opcode = gds ? IR::ValueOpcode::LoadGdsU16 : IR::ValueOpcode::LoadSharedU16; + bits = 16u; + sign = inst.op == IR::Opcode::DsReadSshort; + } else { + opcode = gds ? IR::ValueOpcode::LoadGdsU32 : IR::ValueOpcode::LoadSharedU32; + bits = 32u; + sign = false; + } + const auto loaded = + ir.Emit(opcode, {resource, shared_address(0), ir.GetExec()}, AddMemoryInfo(inst)); + WriteOperand(inst.dst, bits == 32u ? loaded : WidenSubdword(loaded, bits, sign)); + return true; + } + case IR::Opcode::DsWriteByte: + case IR::Opcode::DsWriteShort: + case IR::Opcode::DsWriteB32: { + const auto data = ReadU32(inst.src[0]); + const auto address = shared_address(1); + IR::ValueOpcode opcode; + IR::Value value; + if (inst.op == IR::Opcode::DsWriteByte) { + opcode = gds ? IR::ValueOpcode::WriteGdsU8 : IR::ValueOpcode::WriteSharedU8; + value = NarrowSubdword(data, 8u); + } else if (inst.op == IR::Opcode::DsWriteShort) { + opcode = gds ? IR::ValueOpcode::WriteGdsU16 : IR::ValueOpcode::WriteSharedU16; + value = NarrowSubdword(data, 16u); + } else { + opcode = gds ? IR::ValueOpcode::WriteGdsU32 : IR::ValueOpcode::WriteSharedU32; + value = data; + } + ir.Emit(opcode, {resource, address, value, ir.GetExec()}, AddMemoryInfo(inst)); + return true; + } + case IR::Opcode::DsMinF32: + case IR::Opcode::DsMaxF32: { + const auto opcode = + gds ? (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::GdsAtomicFMin32 + : IR::ValueOpcode::GdsAtomicFMax32) + : (inst.op == IR::Opcode::DsMinF32 ? IR::ValueOpcode::SharedAtomicFMin32 + : IR::ValueOpcode::SharedAtomicFMax32); + ir.Emit(opcode, + {resource, shared_address(1), ReadU32(inst.src[0]), ReadU32(inst.src[2]), + ir.GetExec()}, + AddMemoryInfo(inst)); + return true; + } + case IR::Opcode::DsAppend: + case IR::Opcode::DsConsume: { + const bool append = inst.op == IR::Opcode::DsAppend; + const auto opcode = + gds ? (append ? IR::ValueOpcode::GdsDataAppend : IR::ValueOpcode::GdsDataConsume) + : (append ? IR::ValueOpcode::DataAppend : IR::ValueOpcode::DataConsume); + WriteOperand(inst.dst, ir.Emit(opcode, + {resource, ReadU32(inst.src[0]), ir.GetExec(), + ir.GetExecLo(), ir.GetExecHi()}, + AddMemoryInfo(inst))); + return true; + } + case IR::Opcode::DsWriteAddtidB32: + case IR::Opcode::DsReadAddtidB32: { + const auto m0_index = inst.op == IR::Opcode::DsWriteAddtidB32 ? 1u : 0u; + const auto base = + ir.BitwiseAnd(ReadU32(inst.src[m0_index]), IR::U32(IR::Value(0xffffu))); + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + const auto address = ir.IAdd(ir.IAdd(base, IR::U32(IR::Value(inst.memory.offset))), + ir.ShiftLeftLogical(lane, IR::U32(IR::Value(2u)))); + if (inst.op == IR::Opcode::DsWriteAddtidB32) { + ir.Emit(IR::ValueOpcode::WriteSharedU32, + {resource, address, ReadU32(inst.src[0]), ir.GetExec()}, + AddMemoryInfo(inst)); + } else { + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::LoadSharedU32, + {resource, address, ir.GetExec()}, AddMemoryInfo(inst))); + } + return true; + } + case IR::Opcode::DsSwizzleB32: + WriteOperand(inst.dst, + ir.Emit(IR::ValueOpcode::SwizzleU32, + {ReadU32(inst.src[0]), ReadU32(inst.src[1]), ir.GetExec()})); + return true; + default: return false; + } +} + +bool Translator::TranslateMemoryOperation(const IR::Instruction& inst, + const BufferAddressValues* address_snapshot) { + switch (inst.op) { + case IR::Opcode::LoadSrtDword: + case IR::Opcode::SLoadDword: + case IR::Opcode::SBufferLoadDword: return TranslateScalarMemory(inst); + + case IR::Opcode::BufferLoadUbyte: + case IR::Opcode::BufferLoadSbyte: + case IR::Opcode::BufferLoadUshort: + case IR::Opcode::BufferLoadSshort: + case IR::Opcode::BufferLoadDword: return TranslateBufferLoad(inst, address_snapshot); + + case IR::Opcode::BufferStoreByte: + case IR::Opcode::BufferStoreShort: + case IR::Opcode::BufferStoreDword: return TranslateBufferStore(inst); + + case IR::Opcode::AtomicSwapU32: + case IR::Opcode::AtomicAddU32: + case IR::Opcode::AtomicSubU32: + case IR::Opcode::AtomicSMinI32: + case IR::Opcode::AtomicUMinU32: + case IR::Opcode::AtomicSMaxI32: + case IR::Opcode::AtomicUMaxU32: + case IR::Opcode::AtomicAndU32: + case IR::Opcode::AtomicOrU32: + case IR::Opcode::AtomicXorU32: + case IR::Opcode::AtomicFMinF32: + case IR::Opcode::AtomicFMaxF32: return TranslateAtomicMemory(inst); + + case IR::Opcode::FlatLoadUbyte: + case IR::Opcode::FlatLoadSbyte: + case IR::Opcode::FlatLoadUshort: + case IR::Opcode::FlatLoadSshort: + case IR::Opcode::FlatLoadDword: return TranslateFlatLoad(inst); + + case IR::Opcode::FlatStoreByte: + case IR::Opcode::FlatStoreShort: + case IR::Opcode::FlatStoreDword: return TranslateFlatStore(inst); + + case IR::Opcode::ImageGetResinfo: + case IR::Opcode::ImageGetLod: + case IR::Opcode::ImageLoad: + case IR::Opcode::ImageStore: + case IR::Opcode::ImageSample: + case IR::Opcode::ImageGather4: return TranslateImageMemory(inst); + + case IR::Opcode::DsReadUbyte: + case IR::Opcode::DsReadSbyte: + case IR::Opcode::DsReadUshort: + case IR::Opcode::DsReadSshort: + case IR::Opcode::DsReadB32: + case IR::Opcode::DsWriteByte: + case IR::Opcode::DsWriteShort: + case IR::Opcode::DsWriteB32: + case IR::Opcode::DsMinF32: + case IR::Opcode::DsMaxF32: + case IR::Opcode::DsSwizzleB32: + case IR::Opcode::DsConsume: + case IR::Opcode::DsAppend: + case IR::Opcode::DsWriteAddtidB32: + case IR::Opcode::DsReadAddtidB32: return TranslateSharedMemory(inst); + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.cpp b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp new file mode 100644 index 000000000..82d069db5 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.cpp @@ -0,0 +1,927 @@ +#include "graphics/shader/recompiler/frontend/translate/Translator.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::Frontend { +namespace Detail { + +bool Fail(std::string* error, const std::string& message) { + if (error != nullptr) { + *error = message; + } + return false; +} + +IR::Operand Translator::OffsetOperand(const IR::Operand& operand, uint32_t offset) { + if (offset == 0 || operand.kind != IR::OperandKind::Register) { + return operand; + } + auto result = operand; + result.reg.index += offset; + result.sdwa_sel = 6; + result.sdwa_dst_unused = 2; + result.omod = 0; + result.sdwa_sext = false; + result.op_sel = false; + result.op_sel_hi = false; + result.negate = false; + result.negate_hi = false; + result.absolute = false; + result.clamp = false; + result.dpp_ctrl = 0; + result.dpp_row_mask = 0xf; + result.dpp_bank_mask = 0xf; + result.dpp_fetch_inactive = false; + result.dpp_bound_ctrl = false; + result.dpp = false; + return result; +} + +IR::Operand Translator::PlainOperand(const IR::Operand& operand) { + auto result = operand; + result.sdwa_sel = 6; + result.sdwa_dst_unused = 2; + result.omod = 0; + result.sdwa_sext = false; + result.op_sel = false; + result.op_sel_hi = false; + result.negate = false; + result.negate_hi = false; + result.absolute = false; + result.clamp = false; + result.dpp_ctrl = 0; + result.dpp_row_mask = 0xf; + result.dpp_bank_mask = 0xf; + result.dpp_fetch_inactive = false; + result.dpp_bound_ctrl = false; + result.dpp = false; + return result; +} + +IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) { + EXIT_IF(operand.kind != IR::OperandKind::PcRelativeU32 && + operand.kind != IR::OperandKind::PcRelativeHighU32); + const auto address = + ir.Emit(IR::ValueOpcode::IAdd64, {ir.Emit(IR::ValueOpcode::GetShaderBase), + IR::Value(static_cast(operand.imm))}); + const auto pair = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {address}); + return IR::U32( + ir.Emit(IR::ValueOpcode::CompositeExtractU32x2, + {pair, IR::Value(operand.kind == IR::OperandKind::PcRelativeU32 ? 0u : 1u)})); +} + +std::array Translator::BallotMask(IR::U1 value) { + if (current_per_invocation_masks) { + return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))), + IR::U32(IR::Value(0u))}; + } + const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value}); + const auto low = + IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)})); + // RDNA2 wave32 masks have one architectural dword; the upper half is unused. + const auto high = + current_wave_size == 32u + ? IR::U32(IR::Value(0u)) + : IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)})); + return {low, high}; +} + +IR::U32 Translator::ReadRawU32(const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::ImmediateU32) { + return IR::U32(IR::Value(operand.imm)); + } + if (operand.kind == IR::OperandKind::PcRelativeU32 || + operand.kind == IR::OperandKind::PcRelativeHighU32) { + return ReadPcRelativeU32(operand); + } + if (operand.kind == IR::OperandKind::Null) { + return IR::U32(IR::Value(0u)); + } + EXIT_IF(operand.kind != IR::OperandKind::Register); + switch (operand.reg.file) { + case IR::RegisterFile::Scalar: + return ir.GetScalarReg(static_cast(operand.reg.index)); + case IR::RegisterFile::Vector: + return ir.GetVectorReg(static_cast(operand.reg.index)); + case IR::RegisterFile::Vcc: return operand.reg.index == 0 ? ir.GetVccLo() : ir.GetVccHi(); + case IR::RegisterFile::M0: return ir.GetM0(); + case IR::RegisterFile::Exec: + return operand.reg.index == 0 ? ir.GetExecLo() : ir.GetExecHi(); + case IR::RegisterFile::Scc: + return ir.Select(ir.GetScc(), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); + } + EXIT("invalid register file used as a raw U32 source"); +} + +// Scalar operands share one encoded namespace with VCC, M0, and EXEC aliases. +IR::U32 Translator::ReadScalarCode(uint32_t code) { + if (code < 106u) { + return ir.GetScalarReg(static_cast(code)); + } + switch (code) { + case 106u: return ir.GetVccLo(); + case 107u: return ir.GetVccHi(); + case 124u: return ir.GetM0(); + case 126u: + case 127u: { + const auto mask = BallotMask(ir.GetExec()); + return mask[code - 126u]; + } + default: return IR::U32(IR::Value(0u)); + } +} + +IR::U32 Translator::ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value) { + if (operand.dpp) { + const IR::DppMoveFlags flags { + .control = static_cast(operand.dpp_ctrl), + .row_mask = static_cast(operand.dpp_row_mask), + .bank_mask = static_cast(operand.dpp_bank_mask), + .fetch_inactive = operand.dpp_fetch_inactive, + .bound_control = operand.dpp_bound_ctrl, + }; + value = IR::U32(ir.Emit(IR::ValueOpcode::DppMoveU32, {value, ir.GetExec()}, flags)); + } + if (operand.sdwa_sel != 6u) { + uint32_t offset = 0; + uint32_t width = 0; + if (operand.sdwa_sel <= 3u) { + offset = operand.sdwa_sel * 8u; + width = 8u; + } else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { + offset = operand.sdwa_sel == 5u ? 16u : 0u; + width = 16u; + } else { + EXIT("invalid SDWA source selector"); + } + const auto opcode = operand.sdwa_sext ? IR::ValueOpcode::BitFieldSExtract + : IR::ValueOpcode::BitFieldUExtract; + value = IR::U32(ir.Emit(opcode, {value, IR::Value(offset), IR::Value(width)})); + } + return value; +} + +IR::Value Translator::ReadOperand(const IR::Operand& operand, IR::Type type) { + if (type == IR::Type::U16) { + return ir.Emit(IR::ValueOpcode::ConvertU16U32, + {ApplyBitSourceModifiers(operand, ReadRawU32(operand))}); + } + if (type == IR::Type::F16) { + const auto bits = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, + {ApplyBitSourceModifiers(operand, ReadRawU32(operand))})); + return ir.Emit(IR::ValueOpcode::BitCastF16U16, {bits}); + } + if (type == IR::Type::U1) { + EXIT_IF(operand.kind != IR::OperandKind::Register); + switch (operand.reg.file) { + case IR::RegisterFile::Scc: return ir.GetScc(); + case IR::RegisterFile::Exec: return ir.GetExec(); + case IR::RegisterFile::Vcc: return ir.GetVcc(); + default: break; + } + return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u))); + } + if (type == IR::Type::U64) { + if (operand.kind == IR::OperandKind::Register && + operand.reg.file == IR::RegisterFile::Exec) { + return ir.PackUint2x32(ir.GetExecLo(), ir.GetExecHi()); + } + if (operand.kind == IR::OperandKind::Register && + operand.reg.file == IR::RegisterFile::Vcc) { + return ir.PackUint2x32(ir.GetVccLo(), ir.GetVccHi()); + } + const auto low = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); + IR::U32 high(IR::Value(0u)); + if (operand.kind == IR::OperandKind::Register) { + high = ReadRawU32(OffsetOperand(operand, 1)); + } else if (operand.kind == IR::OperandKind::ImmediateU32 && operand.sext_64) { + high = IR::U32(IR::Value(0xffffffffu)); + } + return ir.PackUint2x32(low, high); + } + auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand)); + if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) { + auto value = ir.BitCastF32(bits); + if (operand.absolute) { + value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value})); + } + if (operand.negate) { + value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value})); + } + return value; + } + if (operand.absolute) { + bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7fffffffu))); + } + if (operand.negate) { + bits = ir.BitwiseXor(bits, IR::U32(IR::Value(0x80000000u))); + } + if (!TypesOverlap(type, IR::Type::U32)) { + EXIT("opcode %s at 0x%08x requested unsupported operand type %s", + IR::OpcodeName(current_opcode).data(), current_pc, IR::TypeName(type).c_str()); + } + return bits; +} + +void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) { + if (operand.kind == IR::OperandKind::Null) { + return; + } + EXIT_IF(operand.kind != IR::OperandKind::Register); + if (operand.sdwa_sel != 6u) { + uint32_t offset = 0; + uint32_t width = 0; + if (operand.sdwa_sel <= 3u) { + offset = operand.sdwa_sel * 8u; + width = 8u; + } else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { + offset = operand.sdwa_sel == 5u ? 16u : 0u; + width = 16u; + } else { + EXIT("invalid SDWA destination selector"); + } + switch (operand.sdwa_dst_unused) { + case 0: + value = + IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert, + {IR::Value(0u), value, IR::Value(offset), IR::Value(width)})); + break; + case 1: { + const auto extended = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldSExtract, + {value, IR::Value(0u), IR::Value(width)})); + value = ir.ShiftLeftLogical(extended, IR::U32(IR::Value(offset))); + break; + } + case 2: { + const uint32_t field_mask = width == 32u ? 0xffffffffu : (1u << width) - 1u; + const auto inserted = + ir.ShiftLeftLogical(ir.BitwiseAnd(value, IR::U32(IR::Value(field_mask))), + IR::U32(IR::Value(offset))); + const auto cleared = ir.BitwiseAnd(ReadRawU32(PlainOperand(operand)), + IR::U32(IR::Value(~(field_mask << offset)))); + value = ir.BitwiseOr(cleared, inserted); + } break; + default: EXIT("reserved SDWA DST_U mode"); + } + } + switch (operand.reg.file) { + case IR::RegisterFile::Scalar: + ir.SetScalarReg(static_cast(operand.reg.index), value); + break; + case IR::RegisterFile::Vector: { + const auto reg = static_cast(operand.reg.index); + const auto old = ir.GetVectorReg(reg); + if (operand.dpp) { + const IR::DppMoveFlags flags { + .control = static_cast(operand.dpp_ctrl), + .row_mask = static_cast(operand.dpp_row_mask), + .bank_mask = static_cast(operand.dpp_bank_mask), + .fetch_inactive = operand.dpp_fetch_inactive, + .bound_control = operand.dpp_bound_ctrl, + }; + value = IR::U32( + ir.Emit(IR::ValueOpcode::DppUpdateU32, {value, old, ir.GetExec()}, flags)); + } else { + value = ir.Select(ir.GetExec(), value, old); + } + ir.SetVectorReg(reg, value); + break; + } + case IR::RegisterFile::Vcc: + if (operand.reg.index == 0) { + ir.SetVccLo(IR::U32(value)); + ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi())); + } else { + ir.SetVccHi(IR::U32(value)); + ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value))); + } + break; + case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break; + case IR::RegisterFile::Exec: + if (operand.reg.index == 0) { + ir.SetExecLo(IR::U32(value)); + ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi())); + } else { + ir.SetExecHi(IR::U32(value)); + ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value))); + } + break; + case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break; + } +} + +IR::U64 Translator::ExpandWholeQuadMask(IR::U64 value) { + const auto shifted1 = + IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(1u)})); + const auto shifted2 = + IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(2u)})); + const auto shifted3 = + IR::U64(ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {value, IR::Value(3u)})); + const auto merged1 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {value, shifted1}); + const auto merged2 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {merged1, shifted2}); + const auto merged3 = ir.Emit(IR::ValueOpcode::BitwiseOr64, {merged2, shifted3}); + const auto low_bits = + ir.Emit(IR::ValueOpcode::BitwiseAnd64, {merged3, IR::Value(0x1111111111111111ull)}); + return IR::U64(ir.Emit(IR::ValueOpcode::IMul64, {low_bits, IR::Value(uint64_t {0xfull})})); +} + +IR::F32 Translator::ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value) { + if (operand.omod != 0u) { + float multiplier = 0.5f; + switch (operand.omod) { + case 1u: multiplier = 2.0f; break; + case 2u: multiplier = 4.0f; break; + default: break; + } + value = IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(multiplier)})); + } + if (operand.clamp) { + value = IR::F32(ir.Emit(IR::ValueOpcode::FPSaturate32, {value})); + } + return value; +} + +void Translator::WriteOperand(const IR::Operand& operand, IR::Value value) { + if (operand.kind == IR::OperandKind::Null) { + return; + } + auto type = value.GetType(); + if (type == IR::Type::F32) { + value = ApplyF32ResultModifiers(operand, IR::F32(value)); + type = IR::Type::F32; + } + if (type == IR::Type::Opaque) { + EXIT("opcode %s at 0x%08x produced an untyped value", IR::OpcodeName(current_opcode).data(), + current_pc); + } + if (type == IR::Type::U1) { + EXIT_IF(operand.kind != IR::OperandKind::Register); + switch (operand.reg.file) { + case IR::RegisterFile::Scc: ir.SetScc(IR::U1(value)); return; + case IR::RegisterFile::Exec: { + const auto mask = BallotMask(IR::U1(value)); + ir.SetExec(IR::U1(value)); + ir.SetExecLo(mask[0]); + ir.SetExecHi(mask[1]); + return; + } + case IR::RegisterFile::Vcc: { + const auto mask = BallotMask(IR::U1(value)); + ir.SetVcc(IR::U1(value)); + ir.SetVccLo(mask[0]); + ir.SetVccHi(mask[1]); + return; + } + default: + WriteRawU32(operand, ir.Select(IR::U1(value), IR::U32(IR::Value(1u)), + IR::U32(IR::Value(0u)))); + return; + } + } + if (type == IR::Type::U16) { + WriteRawU32(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {IR::U16(value)}))); + return; + } + if (type == IR::Type::F16) { + const auto bits = IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {value})); + WriteRawU32(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {bits}))); + return; + } + if (type == IR::Type::U64) { + const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value}); + const auto low = ir.CompositeExtract(unpacked, 0); + const auto high = ir.CompositeExtract(unpacked, 1); + if (operand.kind == IR::OperandKind::Register) { + const auto thread_bit = ThreadBit(low, high); + switch (operand.reg.file) { + case IR::RegisterFile::Exec: + ir.SetExec(thread_bit); + ir.SetExecLo(low); + ir.SetExecHi(high); + return; + case IR::RegisterFile::Vcc: + ir.SetVcc(thread_bit); + ir.SetVccLo(low); + ir.SetVccHi(high); + return; + case IR::RegisterFile::Scalar: + ir.SetThreadBitScalarReg(static_cast(operand.reg.index), + thread_bit); + break; + default: break; + } + } + WriteRawU32(operand, low); + WriteRawU32(OffsetOperand(operand, 1), high); + return; + } + if (type == IR::Type::F32) { + WriteRawU32(operand, ir.BitCastU32(IR::F32(value))); + return; + } + EXIT_IF(type != IR::Type::U32); + WriteRawU32(operand, IR::U32(value)); +} + +IR::U32 Translator::PackHalf2x16(IR::F32 low, IR::F32 high) { + const auto pair = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {low, high}); + return IR::U32(ir.Emit(IR::ValueOpcode::PackHalf2x16, {pair})); +} + +void Translator::WriteF16(const IR::Operand& operand, IR::F32 value) { + value = ApplyF32ResultModifiers(operand, value); + const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value})); + const auto bits = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, + {IR::U16(ir.Emit(IR::ValueOpcode::BitCastU16F16, {half}))})); + auto merged = bits; + if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { + merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert, + {ReadRawU32(PlainOperand(operand)), bits, + IR::Value(operand.sdwa_sel == 5u ? 16u : 0u), IR::Value(16u)})); + } + auto raw = operand; + raw.sdwa_sel = 6u; + raw.omod = 0u; + raw.op_sel = false; + raw.clamp = false; + WriteRawU32(raw, merged); +} + +void Translator::WriteU16(const IR::Operand& operand, IR::U32 value) { + auto merged = ir.BitwiseAnd(value, IR::U32(IR::Value(0xffffu))); + if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { + merged = IR::U32(ir.Emit(IR::ValueOpcode::BitFieldInsert, + {ReadRawU32(PlainOperand(operand)), merged, + IR::Value(operand.sdwa_sel == 5u ? 16u : 0u), IR::Value(16u)})); + } + auto raw = operand; + raw.sdwa_sel = 6u; + raw.omod = 0u; + raw.op_sel = false; + raw.clamp = false; + WriteRawU32(raw, merged); +} + +IR::U32 Translator::ReadU32(const IR::Operand& operand) { + return IR::U32(ReadOperand(operand, IR::Type::U32)); +} + +IR::U64 Translator::ReadU64(const IR::Operand& operand) { + return IR::U64(ReadOperand(operand, IR::Type::U64)); +} + +IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane) { + auto raw_operand = operand; + raw_operand.sdwa_sel = 6; + raw_operand.sdwa_sext = false; + const auto bits = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand)); + uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u; + if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { + offset = operand.sdwa_sel == 5u ? 16u : 0u; + } + const auto half_u32 = IR::U32( + ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(16u)})); + const auto half_u16 = IR::U16(ir.Emit(IR::ValueOpcode::ConvertU16U32, {half_u32})); + const auto half = IR::F16(ir.Emit(IR::ValueOpcode::BitCastF16U16, {half_u16})); + auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half})); + if (operand.absolute) { + value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value})); + } + if (high_lane ? operand.negate_hi : operand.negate) { + value = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {value})); + } + return value; +} + +IR::F32 Translator::ReadF16AsF32(const IR::Operand& operand) { + return ReadF16LaneAsF32(operand, false); +} + +IR::F32 Translator::ReadMixF32(const IR::Operand& operand) { + if (operand.op_sel_hi) { + return ReadF16AsF32(operand); + } + auto value_operand = operand; + value_operand.op_sel = false; + value_operand.op_sel_hi = false; + value_operand.negate_hi = false; + return IR::F32(ReadOperand(value_operand, IR::Type::F32)); +} + +IR::U32 Translator::ReadU16LaneRaw(const IR::Operand& operand, bool high_lane) { + auto raw_operand = operand; + raw_operand.sdwa_sel = 6; + raw_operand.sdwa_sext = false; + const auto bits = ApplyBitSourceModifiers(raw_operand, ReadRawU32(operand)); + uint32_t offset = (high_lane ? operand.op_sel_hi : operand.op_sel) ? 16u : 0u; + uint32_t width = 16u; + if (operand.sdwa_sel <= 3u) { + offset = operand.sdwa_sel * 8u; + width = 8u; + } else if (operand.sdwa_sel == 4u || operand.sdwa_sel == 5u) { + offset = operand.sdwa_sel == 5u ? 16u : 0u; + } + return IR::U32( + ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(offset), IR::Value(width)})); +} + +IR::U32 Translator::ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend) { + auto value = ReadU16LaneRaw(operand, high_lane); + if (high_lane ? operand.negate_hi : operand.negate) { + value = ir.BitwiseAnd(ir.ISub(IR::U32(IR::Value(0u)), value), IR::U32(IR::Value(0xffffu))); + } + if (sign_extend || operand.sdwa_sext) { + value = IR::U32( + ir.Emit(IR::ValueOpcode::BitFieldSExtract, {value, IR::Value(0u), IR::Value(16u)})); + } + return value; +} + +IR::U32 Translator::ReadU16AsU32(const IR::Operand& operand, bool sign_extend) { + return ReadU16LaneAsU32(operand, false, sign_extend); +} + +IR::U32 Translator::ReadF16LaneBits(const IR::Operand& operand, bool high_lane) { + auto value = ReadU16LaneRaw(operand, high_lane); + if (operand.absolute) { + value = ir.BitwiseAnd(value, IR::U32(IR::Value(0x7fffu))); + } + if (high_lane ? operand.negate_hi : operand.negate) { + value = ir.BitwiseXor(value, IR::U32(IR::Value(0x8000u))); + } + return value; +} + +std::array Translator::UnpackU64(IR::U64 value) { + const auto unpacked = ir.Emit(IR::ValueOpcode::UnpackUint2x32, {value}); + return {ir.CompositeExtract(unpacked, 0), ir.CompositeExtract(unpacked, 1)}; +} + +IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) { + if (current_per_invocation_masks) { + return ir.INotEqual(low, IR::U32(IR::Value(0u))); + } + const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId)); + auto word = low; + if (current_wave_size == 64u) { + const auto high_lane = + IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)})); + word = ir.Select(high_lane, high, low); + } + const auto bit = + ir.BitwiseAnd(ir.ShiftRightLogical(word, ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)))), + IR::U32(IR::Value(1u))); + return ir.INotEqual(bit, IR::U32(IR::Value(0u))); +} + +IR::U1 Translator::ReadCondition(const IR::Operand& operand) { + return IR::U1(ReadOperand(operand, IR::Type::U1)); +} + +IR::U32 Translator::ConditionBit(const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::ImmediateU32) { + return IR::U32(IR::Value(operand.imm & 1u)); + } + return ir.Select(ReadCondition(operand), IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))); +} + +IR::U1 Translator::ReadMask(const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::ImmediateU32) { + return IR::U1(IR::Value(operand.imm != 0u || operand.sext_64)); + } + EXIT_IF(operand.kind != IR::OperandKind::Register); + switch (operand.reg.file) { + case IR::RegisterFile::Scalar: { + const auto reg = static_cast(operand.reg.index); + const auto high = + current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs + ? ir.GetScalarReg(static_cast(IR::RegIndex(reg) + 1u)) + : IR::U32(IR::Value(0u)); + return ThreadBit(ir.GetScalarReg(reg), high); + } + case IR::RegisterFile::Exec: return ir.GetExec(); + case IR::RegisterFile::Vcc: return ir.GetVcc(); + case IR::RegisterFile::Scc: return ir.GetScc(); + default: return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u))); + } +} + +void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) { + EXIT_IF(operand.kind != IR::OperandKind::Register); + switch (operand.reg.file) { + case IR::RegisterFile::Scalar: { + const auto reg = static_cast(operand.reg.index); + const auto mask = BallotMask(value); + ir.SetThreadBitScalarReg(reg, value); + ir.SetScalarReg(reg, mask[0]); + // A wave32 VALU mask destination must not overwrite the neighboring SGPR. + if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { + ir.SetScalarReg(static_cast(IR::RegIndex(reg) + 1u), mask[1]); + } + return; + } + case IR::RegisterFile::Exec: { + const auto mask = BallotMask(value); + ir.SetExec(value); + ir.SetExecLo(mask[0]); + ir.SetExecHi(mask[1]); + return; + } + case IR::RegisterFile::Vcc: { + const auto mask = BallotMask(value); + ir.SetVcc(value); + ir.SetVccLo(mask[0]); + ir.SetVccHi(mask[1]); + return; + } + case IR::RegisterFile::Scc: ir.SetScc(value); return; + default: + WriteRawU32(operand, ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u)))); + WriteRawU32(OffsetOperand(operand, 1), IR::U32(IR::Value(0u))); + return; + } +} + +void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) { + if (operand.kind != IR::OperandKind::Register || operand.reg.file != IR::RegisterFile::Scalar) { + WriteMask(operand, value); + return; + } + const auto reg = static_cast(operand.reg.index); + const auto mask = BallotMask(value); + ir.SetThreadBitScalarReg(reg, value); + ir.SetScalarReg(reg, mask[0]); + if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) { + ir.SetScalarReg(static_cast(IR::RegIndex(reg) + 1u), mask[1]); + } +} + +void Translator::WriteCompareResult(const IR::Operand& operand, IR::U1 value) { + if (operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc) { + WriteOperand(operand, value); + return; + } + // VALU instructions only write lanes enabled by the incoming EXEC mask. This is + // especially important for V_CMPX: replacing EXEC with the ungated comparison would + // reactivate lanes disabled by an enclosing divergent region. + WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value)); +} + +bool Translator::TranslateBlock(const IR::BasicBlock& source, std::string* error) { + BufferAddressValues buffer_address_snapshot {}; + uint32_t buffer_address_pc = 0; + bool has_buffer_address_snapshot = false; + for (const auto& source_inst: source.instructions) { + current_opcode = source_inst.op; + current_pc = source_inst.pc; + const bool grouped_buffer_load = + IsBufferLoadOperation(source_inst.op) && source_inst.memory.component_count > 1u; + if (grouped_buffer_load && + (!has_buffer_address_snapshot || buffer_address_pc != source_inst.pc || + source_inst.memory.component_index == 0u)) { + buffer_address_snapshot = ReadBufferAddress(source_inst, 0); + buffer_address_pc = source_inst.pc; + has_buffer_address_snapshot = true; + } else if (!grouped_buffer_load) { + has_buffer_address_snapshot = false; + } + if (static_cast(source_inst.op) >= static_cast(IR::Opcode::Count)) { + return Fail(error, "value IR input opcode is out of range"); + } + if (TranslateInstruction(source_inst, + grouped_buffer_load ? &buffer_address_snapshot : nullptr)) { + continue; + } + return Fail(error, fmt::format("opcode {} at 0x{:08x} has no typed Value IR lowering", + IR::OpcodeName(source_inst.op), source_inst.pc)); + } + return true; +} + +bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info, + std::string* error) { + if (source.terminator.goto_value >= 0) { + if (source.terminator.goto_variable == UINT32_MAX) { + return Fail(error, fmt::format("block {} sets an invalid goto variable", source.id)); + } + ir.SetGotoVariable(source.terminator.goto_variable, + IR::U1(IR::Value(source.terminator.goto_value != 0))); + } + if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) { + if (source.terminator.indirect_selector_code != UINT32_MAX) { + info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code); + } else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) { + info.indirect_target = + ir.GetScalarReg(static_cast(source.terminator.indirect_pc_sgpr)); + } else { + return Fail(error, fmt::format("block {} has no indirect branch selector", source.id)); + } + ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target}); + return true; + } + if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) { + return true; + } + IR::U1 condition; + const auto mask_non_zero = [&](IR::U1 value) { + const auto mask = BallotMask(value); + return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))); + }; + switch (source.terminator.condition) { + case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break; + case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break; + case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break; + case CFG::BranchCondition::VccZero: + condition = ir.LogicalNot(mask_non_zero(ir.GetVcc())); + break; + case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break; + case CFG::BranchCondition::ExecZero: + condition = ir.LogicalNot(mask_non_zero(ir.GetExec())); + break; + case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break; + case CFG::BranchCondition::GotoVariable: + if (source.terminator.goto_variable == UINT32_MAX) { + return Fail(error, + fmt::format("block {} reads an invalid goto variable", source.id)); + } + condition = ir.GetGotoVariable(source.terminator.goto_variable); + break; + case CFG::BranchCondition::Unknown: + return Fail(error, fmt::format("block {} has an unknown branch condition", source.id)); + } + info.condition = condition; + ir.Emit(IR::ValueOpcode::Reference, {condition}); + return true; +} + +} // namespace Detail + +bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, + const ShaderVertexInputInfo* vertex_input_info, + const ShaderPixelInputInfo* pixel_input_info, + const ShaderComputeInputInfo* compute_input_info, std::string* error) { + result = {}; + result.dispatcher_fallback = source.dispatcher_fallback; + const uint32_t wave_size = source.wave_size; + const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; + uint32_t vector_limit = 1u; + const auto include_vector = [&](const IR::Operand& operand) { + if (operand.kind == IR::OperandKind::Register && + operand.reg.file == IR::RegisterFile::Vector) { + vector_limit = + std::min(IR::NumVectorRegs, std::max(vector_limit, operand.reg.index + 1u)); + } + }; + for (const auto& block: source.blocks) { + for (const auto& inst: block.instructions) { + include_vector(inst.dst); + include_vector(inst.dst2); + for (uint32_t index = 0; index < inst.src_count; index++) { + include_vector(inst.src[index]); + } + } + } + result.block_storage.reserve(source.blocks.size() + 1u); + result.blocks.reserve(source.blocks.size() + 1u); + result.block_info.reserve(source.blocks.size() + 1u); + if (!source.blocks.empty()) { + const auto max_id = + std::ranges::max_element(source.blocks, {}, [](const IR::BasicBlock& block) { + return block.id; + })->id; + if (max_id == UINT32_MAX) { + return Detail::Fail(error, "cannot allocate a typed prologue block id"); + } + CFG::Terminator terminator; + terminator.kind = CFG::TerminatorKind::Branch; + terminator.true_block = source.blocks.front().id; + result.block_storage.push_back(std::make_unique()); + result.blocks.push_back(result.block_storage.back().get()); + result.block_info.push_back({max_id + 1u, source.blocks.front().start_pc, + source.blocks.front().start_pc, std::move(terminator)}); + } + for (const auto& source_block: source.blocks) { + result.block_storage.push_back(std::make_unique()); + result.blocks.push_back(result.block_storage.back().get()); + result.block_info.push_back( + {source_block.id, source_block.start_pc, source_block.end_pc, source_block.terminator}); + } + for (size_t index = 0; index < source.blocks.size(); index++) { + for (const auto successor: source.blocks[index].successors) { + if (successor >= source.blocks.size()) { + return Detail::Fail(error, "value IR block successor is out of range"); + } + result.blocks[index + 1u]->AddBranch(result.blocks[successor + 1u]); + } + } + if (!result.blocks.empty()) { + result.blocks.front()->AddBranch(result.blocks[1]); + IR::IREmitter prologue(result.blocks.front()); + const auto builtin = [&](IR::StageInputKind kind, uint32_t component = 0u) { + return IR::U32( + prologue.Emit(IR::ValueOpcode::GetBuiltin, + {IR::Value(static_cast(kind)), IR::Value(component)})); + }; + prologue.Emit(IR::ValueOpcode::Prologue); + const IR::U32 zero(IR::Value(0u)); + for (uint32_t index = 0; index < IR::NumScalarRegs; index++) { + prologue.SetScalarReg(static_cast(index), zero); + } + for (uint32_t index = 0; index < vector_limit; index++) { + prologue.SetVectorReg(static_cast(index), zero); + } + for (uint32_t index = 0; index < source.user_data_count; index++) { + const auto reg = static_cast(source.user_data_base + index); + if (IR::RegIndex(reg) >= IR::NumScalarRegs) { + break; + } + prologue.SetScalarReg(reg, prologue.GetUserData(reg)); + } + prologue.SetScc(IR::U1(IR::Value(false))); + prologue.SetExec(IR::U1(IR::Value(true))); + prologue.SetExecLo(IR::U32(IR::Value(0xffffffffu))); + prologue.SetExecHi(IR::U32(IR::Value(source.wave_size > 32u ? 0xffffffffu : 0u))); + prologue.SetVcc(IR::U1(IR::Value(false))); + prologue.SetVccLo(IR::U32(IR::Value(0u))); + prologue.SetVccHi(IR::U32(IR::Value(0u))); + prologue.SetM0(IR::U32(IR::Value(0u))); + if (source.stage == ShaderType::Compute && compute_input_info != nullptr) { + const auto* cs = compute_input_info; + const auto thread_ids = + cs->thread_ids_num > 0 ? std::min(cs->thread_ids_num, 3u) : 0u; + for (uint32_t index = 0; index < thread_ids; index++) { + prologue.SetVectorReg(static_cast(index), + builtin(IR::StageInputKind::LocalInvocationId, index)); + } + uint32_t reg_offset = 0; + for (uint32_t index = 0; index < 3u; index++) { + if (cs->group_id[index]) { + prologue.SetScalarReg( + static_cast(cs->workgroup_register + reg_offset++), + builtin(IR::StageInputKind::WorkgroupId, index)); + } + } + if (cs->tg_size_en) { + const auto wave_size = cs->wave_size != 0u ? cs->wave_size : 64u; + const auto total_threads = std::max(cs->threads_num[0], 1u) * + std::max(cs->threads_num[1], 1u) * + std::max(cs->threads_num[2], 1u); + const auto waves = std::min((total_threads + wave_size - 1u) / wave_size, 0x3fu); + const auto local_index = builtin(IR::StageInputKind::LocalInvocationIndex); + const auto wave_id = IR::U32( + prologue.Emit(IR::ValueOpcode::UDiv32, {local_index, IR::Value(wave_size)})); + const auto wave_bits = prologue.ShiftLeftLogical(wave_id, IR::U32(IR::Value(20u))); + const auto first_bit = + prologue.Select(prologue.IEqual(wave_id, IR::U32(IR::Value(0u))), + IR::U32(IR::Value(0x80000000u)), IR::U32(IR::Value(0u))); + prologue.SetScalarReg( + static_cast(cs->workgroup_register + reg_offset), + prologue.BitwiseOr(prologue.BitwiseOr(wave_bits, IR::U32(IR::Value(waves))), + first_bit)); + } + } else if (source.stage == ShaderType::Pixel && pixel_input_info != nullptr) { + const auto* ps = pixel_input_info; + uint32_t reg = ps->ps_system_input_base; + if (ps->ps_pos_x) { + prologue.SetVectorReg(static_cast(reg++), + builtin(IR::StageInputKind::FragCoord, 0)); + } + if (ps->ps_pos_y) { + prologue.SetVectorReg(static_cast(reg++), + builtin(IR::StageInputKind::FragCoord, 1)); + } + if (ps->ps_pos_z) { + prologue.SetVectorReg(static_cast(reg++), + builtin(IR::StageInputKind::FragCoord, 2)); + } + if (ps->ps_pos_w) { + prologue.SetVectorReg(static_cast(reg++), + builtin(IR::StageInputKind::FragCoord, 3)); + } + if (ps->ps_front_face) { + prologue.SetVectorReg(static_cast(reg), + builtin(IR::StageInputKind::FrontFacing)); + } + } else if (source.stage == ShaderType::Vertex && vertex_input_info != nullptr) { + prologue.SetVectorReg(static_cast(5), + builtin(IR::StageInputKind::VertexIndex)); + prologue.SetVectorReg(static_cast(8), + builtin(IR::StageInputKind::InstanceIndex)); + } + } + for (size_t index = 0; index < source.blocks.size(); index++) { + Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size, + per_invocation_masks); + if (!translator.TranslateBlock(source.blocks[index], error) || + !translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u], + error)) { + return false; + } + } + return IR::ValidateValueProgram(result, false, error); +} + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Translate.h b/src/graphics/shader/recompiler/frontend/translate/Translate.h new file mode 100644 index 000000000..3c3efb7d0 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Translate.h @@ -0,0 +1,12 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +namespace Libs::Graphics::ShaderRecompiler::Frontend { + +bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result, + const ShaderVertexInputInfo* vertex_input_info, + const ShaderPixelInputInfo* pixel_input_info, + const ShaderComputeInputInfo* compute_input_info, std::string* error); + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend diff --git a/src/graphics/shader/recompiler/frontend/translate/Translator.h b/src/graphics/shader/recompiler/frontend/translate/Translator.h new file mode 100644 index 000000000..71852fec8 --- /dev/null +++ b/src/graphics/shader/recompiler/frontend/translate/Translator.h @@ -0,0 +1,140 @@ +#pragma once + +#include "graphics/shader/recompiler/frontend/translate/Translate.h" +#include "graphics/shader/recompiler/ir/IREmitter.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail { + +struct BufferAddressValues { + IR::U32 index; + IR::U32 offset; + IR::U32 soffset; +}; + +bool Fail(std::string* error, const std::string& message); + +class Translator { +public: + Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit, + uint32_t wave_size, bool per_invocation_masks) + : value_program(program), ir(block), current_vector_limit(vector_limit), + current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {} + + bool TranslateBlock(const IR::BasicBlock& source, std::string* error); + bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info, + std::string* error); + +private: + IR::Operand OffsetOperand(const IR::Operand& operand, uint32_t offset); + IR::Operand PlainOperand(const IR::Operand& operand); + IR::U32 ReadPcRelativeU32(const IR::Operand& operand); + std::array BallotMask(IR::U1 value); + IR::U32 ReadRawU32(const IR::Operand& operand); + IR::U32 ReadScalarCode(uint32_t code); + IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value); + IR::Value ReadOperand(const IR::Operand& operand, IR::Type type); + IR::U1 ThreadBit(IR::U32 low, IR::U32 high); + void WriteRawU32(const IR::Operand& operand, IR::U32 value); + IR::U64 ExpandWholeQuadMask(IR::U64 value); + IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value); + void WriteOperand(const IR::Operand& operand, IR::Value value); + IR::U32 PackHalf2x16(IR::F32 low, IR::F32 high); + void WriteF16(const IR::Operand& operand, IR::F32 value); + void WriteU16(const IR::Operand& operand, IR::U32 value); + IR::U32 ReadU32(const IR::Operand& operand); + IR::U64 ReadU64(const IR::Operand& operand); + IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane); + IR::F32 ReadF16AsF32(const IR::Operand& operand); + IR::F32 ReadMixF32(const IR::Operand& operand); + IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane); + IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend); + IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend); + IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane); + std::array UnpackU64(IR::U64 value); + IR::U1 ReadCondition(const IR::Operand& operand); + IR::U32 ConditionBit(const IR::Operand& operand); + IR::U1 ReadMask(const IR::Operand& operand); + void WriteMask(const IR::Operand& operand, IR::U1 value); + void WriteMask64(const IR::Operand& operand, IR::U1 value); + void WriteCompareResult(const IR::Operand& operand, IR::U1 value); + + IR::MemoryFlags AddMemoryInfo(const IR::Instruction& inst); + IR::ExportFlags AddExportInfo(const IR::Instruction& inst); + struct AddressOperands { + IR::Value resource; + IR::Value low; + IR::Value high; + }; + AddressOperands ReadAddressOperands(const IR::Instruction& inst, uint32_t first_source); + IR::U32 GetResourceDword(uint32_t index, uint32_t dword); + IR::Value GetBufferResource(const IR::MemoryInfo& memory); + IR::Value GetAddressResource(IR::Value low, IR::Value high); + IR::Value GetScalarAddressResource(uint32_t base); + IR::Value GetImageResource(const IR::MemoryInfo& memory); + IR::Value GetSamplerResource(const IR::MemoryInfo& memory); + IR::Value GetSharedResource(IR::ResourceKind kind); + IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base); + IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count); + void WriteImageComponents(const IR::Operand& dst, IR::Value value, uint32_t dmask, + uint32_t component_limit); + IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode); + BufferAddressValues ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset); + IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign); + IR::Value NarrowSubdword(IR::U32 value, uint32_t bits); + IR::ValueOpcode BufferAtomicOpcode(IR::Opcode opcode); + IR::ValueOpcode SharedAtomicOpcode(IR::Opcode opcode, bool gds); + bool IsBufferLoadOperation(IR::Opcode opcode); + bool TranslateScalarMemory(const IR::Instruction& inst); + bool TranslateBufferLoad(const IR::Instruction& inst, + const BufferAddressValues* address_snapshot); + bool TranslateBufferStore(const IR::Instruction& inst); + bool TranslateAtomicMemory(const IR::Instruction& inst); + bool TranslateFlatLoad(const IR::Instruction& inst); + bool TranslateFlatStore(const IR::Instruction& inst); + bool TranslateImageMemory(const IR::Instruction& inst); + bool TranslateSharedMemory(const IR::Instruction& inst); + + IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value); + IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper, + uint32_t high_result); + IR::U32 ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound, + float safe_upper, uint32_t lower_result, + uint32_t upper_result); + IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high); + + bool TranslateInstruction(const IR::Instruction& inst, + const BufferAddressValues* address_snapshot); + bool TranslateStateOperation(const IR::Instruction& inst); + bool TranslateControlOperation(const IR::Instruction& inst); + bool TranslateMove(const IR::Instruction& inst); + bool TranslateLaneOperation(const IR::Instruction& inst); + bool TranslateAttributeOperation(const IR::Instruction& inst); + bool TranslateMemoryOperation(const IR::Instruction& inst, + const BufferAddressValues* address_snapshot = nullptr); + bool TranslateIntegerCompare(const IR::Instruction& inst); + bool TranslateInteger16Compare(const IR::Instruction& inst); + bool TranslateFloatCompare(const IR::Instruction& inst); + bool TranslateConversion(const IR::Instruction& inst); + bool TranslateInteger16Operation(const IR::Instruction& inst); + bool TranslatePackedInteger16(const IR::Instruction& inst); + bool TranslatePackedFloat16(const IR::Instruction& inst); + bool TranslateFloat16Operation(const IR::Instruction& inst); + bool TranslateFloatOperation(const IR::Instruction& inst); + bool TranslatePerInvocationU64Mask(const IR::Instruction& inst); + bool TranslateU64MaskOperation(const IR::Instruction& inst); + bool TranslateSimpleInteger(const IR::Instruction& inst); + bool TranslateComposedInteger(const IR::Instruction& inst); + bool TranslateExtendedInteger(const IR::Instruction& inst); + + IR::ValueProgram& value_program; + IR::IREmitter ir; + IR::Opcode current_opcode = IR::Opcode::ControlNop; + uint32_t current_pc = 0; + uint32_t current_vector_limit = 1; + uint32_t current_wave_size = 64; + bool current_per_invocation_masks = false; +}; + +} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail diff --git a/src/graphics/shader/recompiler/ir/Block.cpp b/src/graphics/shader/recompiler/ir/Block.cpp new file mode 100644 index 000000000..a8d1f1237 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Block.cpp @@ -0,0 +1,84 @@ +#include "graphics/shader/recompiler/ir/Block.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +Block::~Block() { + // Instructions own reverse-use links. Unlink every argument while all definitions are still + // alive, then let the list destroy the detached instructions. + for (auto& inst: instructions) { + inst.Invalidate(); + } +} + +Inst& Block::AppendNewInst(ValueOpcode opcode, std::initializer_list args, uint64_t flags) { + return *PrependNewInst(end(), opcode, args, flags); +} + +Block::iterator Block::PrependNewInst(iterator insertion_point, ValueOpcode opcode, + std::initializer_list args, uint64_t flags) { + const auto expected = NumArgsOf(opcode); + EXIT_IF(expected != std::numeric_limits::max() && expected != args.size()); + const auto inst = instructions.emplace(insertion_point, opcode, flags); + inst->SetParent(this); + size_t index = 0; + for (const auto arg: args) { + inst->SetArg(index++, arg); + } + return inst; +} + +void Block::AddBranch(Block* block) { + EXIT_IF(block == nullptr || std::ranges::find(successors, block) != successors.end()); + EXIT_IF(std::ranges::find(block->predecessors, this) != block->predecessors.end()); + successors.push_back(block); + block->predecessors.push_back(this); +} + +Block::InstructionList& Block::Instructions() { + return instructions; +} + +const Block::InstructionList& Block::Instructions() const { + return instructions; +} + +std::span Block::ImmPredecessors() const { + return predecessors; +} + +std::span Block::ImmSuccessors() const { + return successors; +} + +void Block::SsaSeal() { + ssa_sealed = true; +} + +bool Block::IsSsaSealed() const { + return ssa_sealed; +} + +Block::iterator Block::begin() { + return instructions.begin(); +} + +Block::const_iterator Block::begin() const { + return instructions.begin(); +} + +Block::iterator Block::end() { + return instructions.end(); +} + +Block::const_iterator Block::end() const { + return instructions.end(); +} + +bool Block::empty() const { + return instructions.empty(); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/Block.h b/src/graphics/shader/recompiler/ir/Block.h new file mode 100644 index 000000000..33edbd336 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Block.h @@ -0,0 +1,63 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Reg.h" +#include "graphics/shader/recompiler/ir/Value.h" + +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +// A standard list provides stable instruction addresses without adding an intrusive-container +// dependency. +class Block { +public: + using InstructionList = std::list; + using iterator = InstructionList::iterator; + using const_iterator = InstructionList::const_iterator; + + Block() = default; + ~Block(); + + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + Block(Block&&) = delete; + Block& operator=(Block&&) = delete; + + Inst& AppendNewInst(ValueOpcode opcode, std::initializer_list args = {}, + uint64_t flags = 0); + iterator PrependNewInst(iterator insertion_point, ValueOpcode opcode, + std::initializer_list args = {}, uint64_t flags = 0); + void AddBranch(Block* block); + + [[nodiscard]] InstructionList& Instructions(); + [[nodiscard]] const InstructionList& Instructions() const; + [[nodiscard]] std::span ImmPredecessors() const; + [[nodiscard]] std::span ImmSuccessors() const; + + void SsaSeal(); + [[nodiscard]] bool IsSsaSealed() const; + + iterator begin(); + const_iterator begin() const; + iterator end(); + const_iterator end() const; + bool empty() const; + + std::array ssa_sreg_values {}; + std::array ssa_thread_bit_sreg_values {}; + std::array ssa_vreg_values {}; + +private: + InstructionList instructions; + std::vector predecessors; + std::vector successors; + bool ssa_sealed = false; +}; + +using BlockList = std::vector; + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/IREmitter.cpp b/src/graphics/shader/recompiler/ir/IREmitter.cpp new file mode 100644 index 000000000..8e53c4bc4 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/IREmitter.cpp @@ -0,0 +1,218 @@ +#include "graphics/shader/recompiler/ir/IREmitter.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +IREmitter::IREmitter(Block* value_block): block(value_block) { + EXIT_IF(block == nullptr); +} + +void IREmitter::SetBlock(Block* value_block) { + EXIT_IF(value_block == nullptr); + block = value_block; +} + +Value IREmitter::Emit(ValueOpcode opcode, std::initializer_list args, uint64_t flags) { + auto& inst = block->AppendNewInst(opcode, args, flags); + return TypeOf(opcode) == Type::Void ? Value {} : Value(&inst); +} + +U32 IREmitter::GetUserData(ScalarReg reg) { + return U32(Emit(ValueOpcode::GetUserData, {Value(reg)})); +} + +U32 IREmitter::GetScalarReg(ScalarReg reg) { + return U32(Emit(ValueOpcode::GetScalarRegister, {Value(reg)})); +} + +void IREmitter::SetScalarReg(ScalarReg reg, U32 value) { + Emit(ValueOpcode::SetScalarRegister, {Value(reg), value}); +} + +U1 IREmitter::GetThreadBitScalarReg(ScalarReg reg) { + return U1(Emit(ValueOpcode::GetThreadBitScalarRegister, {Value(reg)})); +} + +void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) { + Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value}); +} + +U32 IREmitter::GetVectorReg(VectorReg reg) { + return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)})); +} + +void IREmitter::SetVectorReg(VectorReg reg, U32 value) { + Emit(ValueOpcode::SetVectorRegister, {Value(reg), value}); +} + +U1 IREmitter::GetGotoVariable(uint32_t id) { + return U1(Emit(ValueOpcode::GetGotoVariable, {Value(id)})); +} + +void IREmitter::SetGotoVariable(uint32_t id, U1 value) { + Emit(ValueOpcode::SetGotoVariable, {Value(id), value}); +} + +U1 IREmitter::GetScc() { + return U1(Emit(ValueOpcode::GetScc)); +} + +void IREmitter::SetScc(U1 value) { + Emit(ValueOpcode::SetScc, {value}); +} + +U1 IREmitter::GetExec() { + return U1(Emit(ValueOpcode::GetExec)); +} + +void IREmitter::SetExec(U1 value) { + Emit(ValueOpcode::SetExec, {value}); +} + +U32 IREmitter::GetExecLo() { + return U32(Emit(ValueOpcode::GetExecLo)); +} + +void IREmitter::SetExecLo(U32 value) { + Emit(ValueOpcode::SetExecLo, {value}); +} + +U32 IREmitter::GetExecHi() { + return U32(Emit(ValueOpcode::GetExecHi)); +} + +void IREmitter::SetExecHi(U32 value) { + Emit(ValueOpcode::SetExecHi, {value}); +} + +U1 IREmitter::GetVcc() { + return U1(Emit(ValueOpcode::GetVcc)); +} + +void IREmitter::SetVcc(U1 value) { + Emit(ValueOpcode::SetVcc, {value}); +} + +U32 IREmitter::GetVccLo() { + return U32(Emit(ValueOpcode::GetVccLo)); +} + +void IREmitter::SetVccLo(U32 value) { + Emit(ValueOpcode::SetVccLo, {value}); +} + +U32 IREmitter::GetVccHi() { + return U32(Emit(ValueOpcode::GetVccHi)); +} + +void IREmitter::SetVccHi(U32 value) { + Emit(ValueOpcode::SetVccHi, {value}); +} + +U32 IREmitter::GetM0() { + return U32(Emit(ValueOpcode::GetM0)); +} + +void IREmitter::SetM0(U32 value) { + Emit(ValueOpcode::SetM0, {value}); +} + +F32 IREmitter::BitCastF32(U32 value) { + return F32(Emit(ValueOpcode::BitCastF32U32, {value})); +} + +U32 IREmitter::BitCastU32(F32 value) { + return U32(Emit(ValueOpcode::BitCastU32F32, {value})); +} + +F16 IREmitter::BitCastF16(U32 value) { + const auto half = U16(Emit(ValueOpcode::ConvertU16U32, {value})); + return F16(Emit(ValueOpcode::BitCastF16U16, {half})); +} + +U32 IREmitter::BitCastU32(F16 value) { + const auto half = U16(Emit(ValueOpcode::BitCastU16F16, {value})); + return U32(Emit(ValueOpcode::ConvertU32U16, {half})); +} + +U64 IREmitter::PackUint2x32(U32 low, U32 high) { + const auto pair = Emit(ValueOpcode::CompositeConstructU32x2, {low, high}); + return U64(Emit(ValueOpcode::PackUint2x32, {pair})); +} + +U32 IREmitter::CompositeExtract(Value composite, uint32_t index) { + return U32(Emit(ValueOpcode::CompositeExtractU32x2, {composite, Value(index)})); +} + +U32 IREmitter::IAdd(U32 lhs, U32 rhs) { + return U32(Emit(ValueOpcode::IAdd32, {lhs, rhs})); +} + +U32 IREmitter::ISub(U32 lhs, U32 rhs) { + return U32(Emit(ValueOpcode::ISub32, {lhs, rhs})); +} + +U32 IREmitter::IMul(U32 lhs, U32 rhs) { + return U32(Emit(ValueOpcode::IMul32, {lhs, rhs})); +} + +U32 IREmitter::ShiftLeftLogical(U32 value, U32 shift) { + return U32(Emit(ValueOpcode::ShiftLeftLogical32, {value, shift})); +} + +U32 IREmitter::ShiftRightLogical(U32 value, U32 shift) { + return U32(Emit(ValueOpcode::ShiftRightLogical32, {value, shift})); +} + +U32 IREmitter::ShiftRightArithmetic(U32 value, U32 shift) { + return U32(Emit(ValueOpcode::ShiftRightArithmetic32, {value, shift})); +} + +U32 IREmitter::BitwiseAnd(U32 lhs, U32 rhs) { + return U32(Emit(ValueOpcode::BitwiseAnd32, {lhs, rhs})); +} + +U32 IREmitter::BitwiseOr(U32 lhs, U32 rhs) { + return U32(Emit(ValueOpcode::BitwiseOr32, {lhs, rhs})); +} + +U32 IREmitter::BitwiseXor(U32 lhs, U32 rhs) { + return U32(Emit(ValueOpcode::BitwiseXor32, {lhs, rhs})); +} + +U32 IREmitter::BitwiseNot(U32 value) { + return U32(Emit(ValueOpcode::BitwiseNot32, {value})); +} + +U32 IREmitter::Select(U1 condition, U32 true_value, U32 false_value) { + return U32(Emit(ValueOpcode::SelectU32, {condition, true_value, false_value})); +} + +U1 IREmitter::IEqual(U32 lhs, U32 rhs) { + return U1(Emit(ValueOpcode::IEqual32, {lhs, rhs})); +} + +U1 IREmitter::INotEqual(U32 lhs, U32 rhs) { + return U1(Emit(ValueOpcode::INotEqual32, {lhs, rhs})); +} + +U1 IREmitter::ULessThan(U32 lhs, U32 rhs) { + return U1(Emit(ValueOpcode::ULessThan32, {lhs, rhs})); +} + +U1 IREmitter::UGreaterThan(U32 lhs, U32 rhs) { + return U1(Emit(ValueOpcode::UGreaterThan32, {lhs, rhs})); +} + +U1 IREmitter::LogicalAnd(U1 lhs, U1 rhs) { + return U1(Emit(ValueOpcode::LogicalAnd, {lhs, rhs})); +} + +U1 IREmitter::LogicalOr(U1 lhs, U1 rhs) { + return U1(Emit(ValueOpcode::LogicalOr, {lhs, rhs})); +} + +U1 IREmitter::LogicalNot(U1 value) { + return U1(Emit(ValueOpcode::LogicalNot, {value})); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/IREmitter.h b/src/graphics/shader/recompiler/ir/IREmitter.h new file mode 100644 index 000000000..808c376bd --- /dev/null +++ b/src/graphics/shader/recompiler/ir/IREmitter.h @@ -0,0 +1,83 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Block.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +class IREmitter { +public: + explicit IREmitter(Block* block); + + void SetBlock(Block* block); + + Value Emit(ValueOpcode opcode, std::initializer_list args = {}, uint64_t flags = 0); + + template + requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v) + Value Emit(ValueOpcode opcode, std::initializer_list args, T flags) { + uint64_t bits = 0; + std::memcpy(&bits, &flags, sizeof(flags)); + return Emit(opcode, args, bits); + } + + U32 GetUserData(ScalarReg reg); + U32 GetScalarReg(ScalarReg reg); + void SetScalarReg(ScalarReg reg, U32 value); + U1 GetThreadBitScalarReg(ScalarReg reg); + void SetThreadBitScalarReg(ScalarReg reg, U1 value); + U32 GetVectorReg(VectorReg reg); + void SetVectorReg(VectorReg reg, U32 value); + U1 GetGotoVariable(uint32_t id); + void SetGotoVariable(uint32_t id, U1 value); + U1 GetScc(); + void SetScc(U1 value); + U1 GetExec(); + void SetExec(U1 value); + U32 GetExecLo(); + void SetExecLo(U32 value); + U32 GetExecHi(); + void SetExecHi(U32 value); + U1 GetVcc(); + void SetVcc(U1 value); + U32 GetVccLo(); + void SetVccLo(U32 value); + U32 GetVccHi(); + void SetVccHi(U32 value); + U32 GetM0(); + void SetM0(U32 value); + + F32 BitCastF32(U32 value); + U32 BitCastU32(F32 value); + F16 BitCastF16(U32 value); + U32 BitCastU32(F16 value); + U64 PackUint2x32(U32 low, U32 high); + U32 CompositeExtract(Value composite, uint32_t index); + + U32 IAdd(U32 lhs, U32 rhs); + U32 ISub(U32 lhs, U32 rhs); + U32 IMul(U32 lhs, U32 rhs); + U32 ShiftLeftLogical(U32 value, U32 shift); + U32 ShiftRightLogical(U32 value, U32 shift); + U32 ShiftRightArithmetic(U32 value, U32 shift); + U32 BitwiseAnd(U32 lhs, U32 rhs); + U32 BitwiseOr(U32 lhs, U32 rhs); + U32 BitwiseXor(U32 lhs, U32 rhs); + U32 BitwiseNot(U32 value); + U32 Select(U1 condition, U32 true_value, U32 false_value); + U1 IEqual(U32 lhs, U32 rhs); + U1 INotEqual(U32 lhs, U32 rhs); + U1 ULessThan(U32 lhs, U32 rhs); + U1 UGreaterThan(U32 lhs, U32 rhs); + U1 LogicalAnd(U1 lhs, U1 rhs); + U1 LogicalOr(U1 lhs, U1 rhs); + U1 LogicalNot(U1 value); + +private: + Block* block = nullptr; +}; + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ReadLaneElimination.cpp b/src/graphics/shader/recompiler/ir/ReadLaneElimination.cpp deleted file mode 100644 index fe0adc2a3..000000000 --- a/src/graphics/shader/recompiler/ir/ReadLaneElimination.cpp +++ /dev/null @@ -1,323 +0,0 @@ -#include "graphics/shader/recompiler/ir/ReadLaneElimination.h" - -#include "graphics/shader/recompiler/ir/SrtWalker.h" - -#include -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -namespace { - -constexpr uint32_t FirstTemporaryScalarRegister = 128; - -struct LaneKey { - uint32_t reg = 0; - uint32_t lane = 0; - - auto operator<=>(const LaneKey&) const = default; -}; - -using LaneSet = std::set; - -bool PairDwordOpcode(Opcode op) { - switch (op) { - case Opcode::MoveU64: - case Opcode::WqmB64: - case Opcode::SaveexecB64: - case Opcode::BitwiseAndU64: - case Opcode::BitwiseAndNotU64: - case Opcode::BitwiseOrU64: - case Opcode::BitwiseOrNotU64: - case Opcode::BitwiseXorU64: - case Opcode::BitwiseNandU64: - case Opcode::BitwiseNorU64: - case Opcode::BitwiseXnorU64: - case Opcode::BitwiseNotU64: - case Opcode::BitFieldMaskU64: - case Opcode::BitFieldExtractU64: - case Opcode::BitReplicateB64B32: - case Opcode::ShiftLeftLogicalU64: - case Opcode::ShiftRightLogicalU64: - case Opcode::SelectU64: return true; - default: return false; - } -} - -bool ResolveLane(const Program& program, const Instruction& inst, uint32_t source_index, - uint32_t& lane) { - if (source_index >= inst.src_count || (program.wave_size != 32 && program.wave_size != 64)) { - return false; - } - const auto& selector = inst.src[source_index]; - if (selector.kind == OperandKind::ImmediateU32) { - lane = selector.imm % program.wave_size; - return true; - } - uint32_t folded = 0; - if (!FoldScalarConstant(program.provenance, inst.scalar_sources[source_index], folded)) { - return false; - } - lane = folded % program.wave_size; - return true; -} - -bool UniformWriteSource(const Instruction& inst) { - if (inst.src_count == 0) { - return false; - } - const auto& source = inst.src[0]; - if (source.kind == OperandKind::ImmediateU32 || source.kind == OperandKind::PcRelativeU32) { - return true; - } - return source.kind == OperandKind::Register && - (source.reg.file == RegisterFile::Scalar || source.reg.file == RegisterFile::Scc || - source.reg.file == RegisterFile::M0); -} - -bool WriteLaneKey(const Program& program, const Instruction& inst, LaneKey& key) { - if (inst.op != Opcode::WriteLaneU32 || inst.dst.kind != OperandKind::Register || - inst.dst.reg.file != RegisterFile::Vector || !UniformWriteSource(inst)) { - return false; - } - uint32_t lane = 0; - if (!ResolveLane(program, inst, 1, lane)) { - return false; - } - key = {inst.dst.reg.index, lane}; - return true; -} - -bool ReadLaneKey(const Program& program, const Instruction& inst, LaneKey& key) { - if (inst.op != Opcode::ReadLaneU32 || inst.src_count < 2 || - inst.src[0].kind != OperandKind::Register || inst.src[0].reg.file != RegisterFile::Vector) { - return false; - } - uint32_t lane = 0; - if (!ResolveLane(program, inst, 1, lane)) { - return false; - } - key = {inst.src[0].reg.index, lane}; - return true; -} - -void InvalidateRegister(LaneSet& valid, uint32_t reg) { - const auto first = valid.lower_bound({reg, 0}); - const auto last = valid.lower_bound({reg + 1u, 0}); - valid.erase(first, last); -} - -void ApplyInstruction(const Program& program, const Instruction& inst, LaneSet& valid) { - if (inst.op == Opcode::WriteLaneU32 && inst.dst.kind == OperandKind::Register && - inst.dst.reg.file == RegisterFile::Vector) { - LaneKey key; - if (WriteLaneKey(program, inst, key)) { - valid.insert(key); - return; - } - uint32_t lane = 0; - if (ResolveLane(program, inst, 1, lane)) { - valid.erase({inst.dst.reg.index, lane}); - } else { - InvalidateRegister(valid, inst.dst.reg.index); - } - return; - } - - if (inst.op == Opcode::MoveRelDestU32 && inst.dst.kind == OperandKind::Register && - inst.dst.reg.file == RegisterFile::Vector) { - valid.clear(); - return; - } - - if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Vector) { - uint32_t dwords = std::max(inst.memory.data_dwords, 1u); - if (PairDwordOpcode(inst.op) || inst.op == Opcode::UMadU64U32) { - dwords = std::max(dwords, 2u); - } - for (uint32_t i = 0; i < dwords && inst.dst.reg.index <= UINT32_MAX - i; i++) { - InvalidateRegister(valid, inst.dst.reg.index + i); - } - } - if (inst.dst2.kind == OperandKind::Register && inst.dst2.reg.file == RegisterFile::Vector) { - InvalidateRegister(valid, inst.dst2.reg.index); - } -} - -LaneSet TransferBlock(const Program& program, const BasicBlock& block, LaneSet state) { - for (const auto& inst: block.instructions) { - ApplyInstruction(program, inst, state); - } - return state; -} - -LaneSet Intersect(const LaneSet& left, const LaneSet& right) { - LaneSet result; - std::set_intersection(left.begin(), left.end(), right.begin(), right.end(), - std::inserter(result, result.end())); - return result; -} - -uint32_t NextTemporaryScalarRegister(const Program& program) { - uint32_t next = FirstTemporaryScalarRegister; - const auto consider = [&next](const Operand& operand) { - if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::Scalar && - operand.reg.index >= next && operand.reg.index != UINT32_MAX) { - next = operand.reg.index + 1u; - } - }; - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - consider(inst.dst); - consider(inst.dst2); - for (uint32_t i = 0; i < inst.src_count; i++) { - consider(inst.src[i]); - } - } - } - return next; -} - -Operand ScalarRegisterOperand(uint32_t reg) { - Operand operand; - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Scalar; - operand.reg.index = reg; - return operand; -} - -Instruction ShadowWrite(const Instruction& write, uint32_t temporary) { - Instruction shadow; - shadow.pc = write.pc; - shadow.op = Opcode::MoveU32; - shadow.dst = ScalarRegisterOperand(temporary); - shadow.src[0] = write.src[0]; - shadow.src_count = 1; - return shadow; -} - -Instruction ShadowRead(const Instruction& read, uint32_t temporary) { - Instruction rewritten; - rewritten.pc = read.pc; - rewritten.op = Opcode::MoveU32; - rewritten.dst = read.dst; - rewritten.src[0] = ScalarRegisterOperand(temporary); - rewritten.src_count = 1; - return rewritten; -} - -} // namespace - -ReadLaneEliminationStats EliminateReadLane(Program& program) { - ReadLaneEliminationStats stats; - if (program.blocks.empty() || (program.wave_size != 32 && program.wave_size != 64)) { - return stats; - } - - LaneSet universe; - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - LaneKey key; - if (WriteLaneKey(program, inst, key)) { - universe.insert(key); - } - } - } - if (universe.empty()) { - return stats; - } - - const size_t block_count = program.blocks.size(); - std::vector entry(block_count, universe); - std::vector exit(block_count, universe); - entry[0].clear(); - for (size_t block = 0; block < block_count; block++) { - exit[block] = TransferBlock(program, program.blocks[block], entry[block]); - } - - bool changed = true; - while (changed) { - changed = false; - for (size_t block_index = 0; block_index < block_count; block_index++) { - LaneSet next_entry; - const auto& block = program.blocks[block_index]; - if (block_index != 0 && !block.predecessors.empty()) { - next_entry = universe; - for (const auto predecessor: block.predecessors) { - if (predecessor >= block_count) { - next_entry.clear(); - break; - } - next_entry = Intersect(next_entry, exit[predecessor]); - } - } - auto next_exit = TransferBlock(program, block, next_entry); - if (next_entry != entry[block_index] || next_exit != exit[block_index]) { - entry[block_index] = std::move(next_entry); - exit[block_index] = std::move(next_exit); - changed = true; - } - } - } - - LaneSet forwarded; - for (size_t block_index = 0; block_index < block_count; block_index++) { - auto state = entry[block_index]; - for (const auto& inst: program.blocks[block_index].instructions) { - LaneKey key; - if (ReadLaneKey(program, inst, key) && state.contains(key)) { - forwarded.insert(key); - } - ApplyInstruction(program, inst, state); - } - } - if (forwarded.empty()) { - return stats; - } - - std::map temporaries; - auto next_temporary = NextTemporaryScalarRegister(program); - for (const auto& key: forwarded) { - if (next_temporary == UINT32_MAX) { - return {}; - } - temporaries.emplace(key, next_temporary++); - } - - for (size_t block_index = 0; block_index < block_count; block_index++) { - const auto original = std::move(program.blocks[block_index].instructions); - auto& rewritten = program.blocks[block_index].instructions; - rewritten.clear(); - rewritten.reserve(original.size() + temporaries.size()); - auto state = entry[block_index]; - for (const auto& inst: original) { - LaneKey read_key; - if (ReadLaneKey(program, inst, read_key) && state.contains(read_key)) { - const auto temporary = temporaries.find(read_key); - if (temporary != temporaries.end()) { - rewritten.push_back(ShadowRead(inst, temporary->second)); - stats.rewritten_reads++; - ApplyInstruction(program, inst, state); - continue; - } - } - - rewritten.push_back(inst); - LaneKey write_key; - if (WriteLaneKey(program, inst, write_key)) { - const auto temporary = temporaries.find(write_key); - if (temporary != temporaries.end()) { - rewritten.push_back(ShadowWrite(inst, temporary->second)); - stats.shadow_writes++; - } - } - ApplyInstruction(program, inst, state); - } - } - return stats; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ReadLaneElimination.h b/src/graphics/shader/recompiler/ir/ReadLaneElimination.h deleted file mode 100644 index 4861b351c..000000000 --- a/src/graphics/shader/recompiler/ir/ReadLaneElimination.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_READLANEELIMINATION_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_READLANEELIMINATION_H_ - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -namespace Libs::Graphics::ShaderRecompiler::IR { - -struct ReadLaneEliminationStats { - uint32_t rewritten_reads = 0; - uint32_t shadow_writes = 0; -}; - -// Replaces fixed-lane ReadLane operations that are reached by a matching WriteLane on every -// control-flow path. A synthetic scalar register snapshots the value at WriteLane execution time, -// so the rewrite remains valid when the source SGPR is subsequently overwritten. -[[nodiscard]] ReadLaneEliminationStats EliminateReadLane(Program& program); - -} // namespace Libs::Graphics::ShaderRecompiler::IR - -#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_READLANEELIMINATION_H_ */ diff --git a/src/graphics/shader/recompiler/ir/Reg.h b/src/graphics/shader/recompiler/ir/Reg.h new file mode 100644 index 000000000..ea81eda58 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Reg.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +enum class ScalarReg : uint16_t {}; +enum class VectorReg : uint16_t {}; + +// Architectural SGPRs occupy the low range. The current shader-only migration also accepts the +// scalar shadow registers produced by Kyty's pre-existing read-lane pass; that pass disappears +// once read-lane elimination operates on the SSA graph. +constexpr uint32_t NumScalarRegs = 512; +constexpr uint32_t NumVectorRegs = 256; + +constexpr uint32_t RegIndex(ScalarReg reg) { + return static_cast(reg); +} + +constexpr uint32_t RegIndex(VectorReg reg) { + return static_cast(reg); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/ResourceTracking.cpp deleted file mode 100644 index 244206c9d..000000000 --- a/src/graphics/shader/recompiler/ir/ResourceTracking.cpp +++ /dev/null @@ -1,464 +0,0 @@ -#include "graphics/shader/recompiler/ir/ResourceTracking.h" - -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" - -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -const char* StageName(ShaderType stage) { - switch (stage) { - case ShaderType::Vertex: return "vertex"; - case ShaderType::Pixel: return "pixel"; - case ShaderType::Fetch: return "fetch"; - case ShaderType::Compute: return "compute"; - default: return "unknown"; - } -} - -bool IsAtomic(Opcode op) { - switch (op) { - case Opcode::AtomicSwapU32: - case Opcode::AtomicAddU32: - case Opcode::AtomicSubU32: - case Opcode::AtomicSMinI32: - case Opcode::AtomicUMinU32: - case Opcode::AtomicSMaxI32: - case Opcode::AtomicUMaxU32: - case Opcode::AtomicAndU32: - case Opcode::AtomicOrU32: - case Opcode::AtomicXorU32: - case Opcode::AtomicFMinF32: - case Opcode::AtomicFMaxF32: return true; - default: return false; - } -} - -bool IsWrite(Opcode op) { - switch (op) { - case Opcode::BufferStoreByte: - case Opcode::BufferStoreShort: - case Opcode::BufferStoreDword: - case Opcode::ImageStore: return true; - default: return IsAtomic(op); - } -} - -bool NeedsSampler(Opcode op) { - return op == Opcode::ImageSample || op == Opcode::ImageGather4 || op == Opcode::ImageGetLod; -} - -bool IsDepthCompare(const Instruction& inst) { - return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0; -} - -ImageMipMode MipMode(const Instruction& inst) { - const bool storage = inst.memory.kind == ResourceKind::StorageImage || - inst.memory.kind == ResourceKind::StorageImageUint; - return storage && inst.memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None; -} - -bool IsBuffer(const Instruction& inst) { - return inst.memory.kind == ResourceKind::Buffer || - (inst.memory.kind == ResourceKind::ScalarBuffer && inst.op == Opcode::SBufferLoadDword); -} - -bool IsImage(const Instruction& inst) { - return inst.memory.kind == ResourceKind::Image || inst.memory.kind == ResourceKind::ImageUint || - inst.memory.kind == ResourceKind::StorageImage || - inst.memory.kind == ResourceKind::StorageImageUint; -} - -bool IsAddress(const Instruction& inst) { - return inst.op == Opcode::SLoadDword || inst.memory.kind == ResourceKind::Flat || - inst.memory.kind == ResourceKind::Global || inst.memory.kind == ResourceKind::Scratch; -} - -uint32_t ByteExtent(const Instruction& inst) { - const auto bytes = std::max((inst.memory.data_bits + 7u) / 8u, 1u); - const auto count = std::max(inst.memory.data_dwords, 1u); - const auto end = - static_cast(inst.memory.offset) + static_cast(bytes) * count; - return end > UINT32_MAX ? UINT32_MAX : static_cast(end); -} - -bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vector& visited, - std::vector& path) { - path.push_back(id); - if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) { - return true; - } - if (visited[id] != 0) { - path.pop_back(); - return false; - } - visited[id] = 1; - const auto& value = provenance.values[id]; - if (value.op == ScalarValueOp::Phi) { - for (const auto arg: value.phi_args) { - if (ContainsUnknown(provenance, arg, visited, path)) { - return true; - } - } - path.pop_back(); - return false; - } - const auto args = ScalarValueArgCount(value.op); - for (uint32_t i = 0; i < args; i++) { - if (ContainsUnknown(provenance, value.args[i], visited, path)) { - return true; - } - } - path.pop_back(); - return false; -} - -bool IsLoopInvariantValue(const ScalarProvenance& provenance, uint32_t id, - std::vector& visiting) { - if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) { - return false; - } - if (visiting[id] != 0) { - return true; - } - visiting[id] = 1; - const auto& value = provenance.values[id]; - if (value.op == ScalarValueOp::Phi) { - uint32_t invariant = ScalarProvenance::Undefined; - for (const auto arg: value.phi_args) { - if (arg == id) { - continue; - } - if (invariant == ScalarProvenance::Undefined) { - invariant = arg; - } else if (arg != invariant) { - return false; - } - } - return invariant != ScalarProvenance::Undefined && - IsLoopInvariantValue(provenance, invariant, visiting); - } - for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { - if (!IsLoopInvariantValue(provenance, value.args[i], visiting)) { - return false; - } - } - return true; -} - -bool IsLoopInvariantDescriptor(const ScalarProvenance& provenance, - const DescriptorValue& descriptor) { - std::vector visiting(provenance.values.size()); - for (uint32_t i = 0; i < descriptor.dword_count; i++) { - if (!IsLoopInvariantValue(provenance, descriptor.dwords[i], visiting)) { - return false; - } - } - return true; -} - -class Tracker { -public: - explicit Tracker(Program& program): m_program(program) {} - - bool Run(std::string* error) { - if (m_program.resource_tracking_complete) { - return Fail(0, error, "resources already tracked"); - } - if (!m_program.srt_plan_complete) { - return Fail(0, error, "SRT plan is not ready"); - } - if (!m_program.srt_patching_complete) { - return Fail(0, error, "SRT reads were not patched"); - } - for (auto& block: m_program.blocks) { - for (auto& inst: block.instructions) { - if (!Collect(inst, error)) { - return false; - } - } - } - LinkImageAliases(); - m_program.info = std::move(m_info); - for (const auto& patch: m_patches) { - auto& inst = patch.inst.get(); - inst.memory.resource = patch.resource; - inst.memory.sampler = patch.sampler; - inst.memory.resource_source = ScalarProvenance::Undefined; - inst.memory.sampler_source = ScalarProvenance::Undefined; - } - m_program.resource_tracking_complete = true; - return true; - } - -private: - struct Patch { - std::reference_wrapper inst; - uint32_t resource = 0; - uint32_t sampler = 0; - }; - - bool Fail(uint32_t pc, std::string* error, const std::string& reason) const { - if (error != nullptr) { - *error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}", - m_program.shader_hash, StageName(m_program.stage), pc, reason); - } - return false; - } - - bool ValidateSource(uint32_t source, uint32_t dwords, uint32_t pc, std::string* error) const { - const auto* descriptor = GetDescriptorSource(m_program, source); - if (descriptor == nullptr || descriptor->dword_count != dwords) { - return Fail(pc, error, - fmt::format("descriptor source {} is missing or has wrong width", source)); - } - std::vector visited(m_program.provenance.values.size()); - for (uint32_t i = 0; i < descriptor->dword_count; i++) { - std::vector path; - if (ContainsUnknown(m_program.provenance, descriptor->dwords[i], visited, path)) { - const auto value = descriptor->dwords[i]; - std::string chain; - for (const auto id: path) { - const auto op = id < m_program.provenance.values.size() - ? static_cast(m_program.provenance.values[id].op) - : UINT32_MAX; - chain += fmt::format("{}{}:{}({})", chain.empty() ? "" : " -> ", id, op, - ScalarValueToString(m_program.provenance, id)); - } - return Fail( - pc, error, - fmt::format( - "descriptor source {} dword {} contains an unknown value {} ({}) path {}", - source, i, value, ScalarValueToString(m_program.provenance, value), chain)); - } - } - const auto dynamic = - std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(), - source) != m_program.srt.dynamic_sources.end(); - if (dynamic && !IsLoopInvariantDescriptor(m_program.provenance, *descriptor)) { - std::string detail; - for (uint32_t i = 0; i < descriptor->dword_count; i++) { - const auto id = descriptor->dwords[i]; - detail += - fmt::format(" d{}={}({})", i, id, - id < m_program.provenance.values.size() - ? static_cast(m_program.provenance.values[id].op) - : UINT32_MAX); - if (id < m_program.provenance.values.size()) { - for (const auto arg: m_program.provenance.values[id].phi_args) { - detail += fmt::format( - "/{}({})", arg, - arg < m_program.provenance.values.size() - ? static_cast(m_program.provenance.values[arg].op) - : UINT32_MAX); - } - } - } - return Fail(pc, error, - fmt::format("descriptor source {} requires unsupported GPU selection{}", - source, detail)); - } - if (!DescriptorSourceResolved(m_program, source) && !dynamic) { - return Fail(pc, error, fmt::format("descriptor source {} is unresolved", source)); - } - return true; - } - - uint32_t AddBuffer(const Instruction& inst) { - for (uint32_t i = 0; i < m_info.buffers.size(); i++) { - if (m_info.buffers[i].source == inst.memory.resource_source) { - Merge(&m_info.buffers[i], inst); - return i; - } - } - if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) { - return UINT32_MAX; - } - BufferResource resource; - resource.source = inst.memory.resource_source; - resource.first_use_pc = inst.pc; - Merge(&resource, inst); - m_info.buffers.push_back(resource); - return static_cast(m_info.buffers.size() - 1); - } - - void Merge(BufferResource* resource, const Instruction& inst) const { - resource->first_use_pc = std::min(resource->first_use_pc, inst.pc); - resource->max_byte_extent = std::max(resource->max_byte_extent, ByteExtent(inst)); - resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op); - resource->written = resource->written || IsWrite(inst.op); - resource->atomic = resource->atomic || IsAtomic(inst.op); - resource->formatted = resource->formatted || inst.memory.formatted; - resource->scalar = resource->scalar || inst.memory.kind == ResourceKind::ScalarBuffer; - } - - void LinkImageAliases() { - for (auto& buffer: m_info.buffers) { - const auto* buffer_source = GetDescriptorSource(m_program, buffer.source); - if (buffer_source == nullptr || buffer_source->dword_count != 4) { - continue; - } - for (uint32_t i = 0; i < m_info.images.size(); i++) { - const auto* image_source = GetDescriptorSource(m_program, m_info.images[i].source); - if (image_source != nullptr && image_source->dword_count == 8 && - std::equal(buffer_source->dwords.begin(), buffer_source->dwords.begin() + 4, - image_source->dwords.begin())) { - buffer.image_alias = i; - break; - } - } - } - } - - uint32_t AddImage(const Instruction& inst) { - for (uint32_t i = 0; i < m_info.images.size(); i++) { - auto& image = m_info.images[i]; - if (image.source == inst.memory.resource_source && image.kind == inst.memory.kind && - image.dimension == inst.memory.image_dimension && image.mip_mode == MipMode(inst) && - image.depth_compare == IsDepthCompare(inst)) { - Merge(&image, inst); - return i; - } - } - if (m_info.images.size() >= ShaderInfo::MaxImages) { - return UINT32_MAX; - } - ImageResource image; - image.source = inst.memory.resource_source; - image.first_use_pc = inst.pc; - image.kind = inst.memory.kind; - image.dimension = inst.memory.image_dimension; - image.mip_mode = MipMode(inst); - image.depth_compare = IsDepthCompare(inst); - Merge(&image, inst); - m_info.images.push_back(image); - return static_cast(m_info.images.size() - 1); - } - - uint32_t AddAddress(const Instruction& inst) { - auto immediate = static_cast(inst.memory.offset); - if (inst.memory.kind == ResourceKind::ScalarBuffer) { - immediate = static_cast(static_cast(immediate) & ~3u); - } - const auto min_offset = - inst.memory.resource_source == ScalarProvenance::Unknown ? 0 : std::min(immediate, 0); - for (uint32_t i = 0; i < m_info.addresses.size(); i++) { - auto& address = m_info.addresses[i]; - if (address.source == inst.memory.resource_source && address.kind == inst.memory.kind) { - address.first_use_pc = std::min(address.first_use_pc, inst.pc); - address.min_offset = std::min(address.min_offset, min_offset); - address.read = address.read || !IsWrite(inst.op) || IsAtomic(inst.op); - address.written = address.written || IsWrite(inst.op); - address.atomic = address.atomic || IsAtomic(inst.op); - return i; - } - } - if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) { - return UINT32_MAX; - } - AddressResource address {inst.memory.resource_source, inst.pc, inst.memory.kind, - min_offset}; - address.read = !IsWrite(inst.op) || IsAtomic(inst.op); - address.written = IsWrite(inst.op); - address.atomic = IsAtomic(inst.op); - m_info.addresses.push_back(address); - return static_cast(m_info.addresses.size() - 1); - } - - void Merge(ImageResource* resource, const Instruction& inst) const { - resource->first_use_pc = std::min(resource->first_use_pc, inst.pc); - resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op); - resource->written = resource->written || IsWrite(inst.op); - resource->atomic = resource->atomic || IsAtomic(inst.op); - } - - uint32_t AddSampler(const Instruction& inst) { - for (uint32_t i = 0; i < m_info.samplers.size(); i++) { - if (m_info.samplers[i].source == inst.memory.sampler_source) { - m_info.samplers[i].first_use_pc = - std::min(m_info.samplers[i].first_use_pc, inst.pc); - return i; - } - } - if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) { - return UINT32_MAX; - } - m_info.samplers.push_back({inst.memory.sampler_source, inst.pc}); - return static_cast(m_info.samplers.size() - 1); - } - - bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) { - for (auto& pair: m_info.sampled_pairs) { - if (pair.image == image && pair.sampler == sampler) { - pair.first_use_pc = std::min(pair.first_use_pc, pc); - return true; - } - } - if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) { - return Fail(pc, error, "sampled image/sampler pair limit exceeded"); - } - m_info.sampled_pairs.push_back({image, sampler, pc}); - return true; - } - - bool Collect(Instruction& inst, std::string* error) { - if (IsAddress(inst)) { - const bool unbased = inst.memory.resource_source == ScalarProvenance::Unknown; - if ((!unbased && !ValidateSource(inst.memory.resource_source, 2, inst.pc, error)) || - (unbased && inst.memory.kind != ResourceKind::Flat && - inst.memory.kind != ResourceKind::Global && - inst.memory.kind != ResourceKind::Scratch)) { - return unbased ? Fail(inst.pc, error, "scalar memory base is unresolved") : false; - } - const auto resource = AddAddress(inst); - if (resource == UINT32_MAX) { - return Fail(inst.pc, error, "address resource limit exceeded"); - } - m_patches.push_back({std::ref(inst), resource, 0}); - return true; - } - if (!IsBuffer(inst) && !IsImage(inst)) { - return true; - } - if (!ValidateSource(inst.memory.resource_source, IsBuffer(inst) ? 4u : 8u, inst.pc, - error)) { - return false; - } - const auto resource = IsBuffer(inst) ? AddBuffer(inst) : AddImage(inst); - if (resource == UINT32_MAX) { - return Fail(inst.pc, error, - IsBuffer(inst) ? "buffer resource limit exceeded" - : "image resource limit exceeded"); - } - uint32_t sampler = 0; - if (NeedsSampler(inst.op)) { - if (!ValidateSource(inst.memory.sampler_source, 4, inst.pc, error)) { - return false; - } - sampler = AddSampler(inst); - if (sampler == UINT32_MAX) { - return Fail(inst.pc, error, "sampler resource limit exceeded"); - } - if (!AddSampledPair(resource, sampler, inst.pc, error)) { - return false; - } - } - m_patches.push_back({std::ref(inst), resource, sampler}); - return true; - } - - Program& m_program; - ShaderInfo m_info; - std::vector m_patches; -}; - -} // namespace - -bool TrackResources(Program& program, std::string* error) { - return Tracker(program).Run(error); -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ScalarProvenance.cpp b/src/graphics/shader/recompiler/ir/ScalarProvenance.cpp deleted file mode 100644 index e27bc8cc8..000000000 --- a/src/graphics/shader/recompiler/ir/ScalarProvenance.cpp +++ /dev/null @@ -1,963 +0,0 @@ -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" - -#include -#include -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -uint32_t ScalarValueArgCount(ScalarValueOp op) { - switch (op) { - case ScalarValueOp::Not: return 1; - case ScalarValueOp::Add: - case ScalarValueOp::Sub: - case ScalarValueOp::Mul: - case ScalarValueOp::And: - case ScalarValueOp::AndNot: - case ScalarValueOp::Or: - case ScalarValueOp::OrNot: - case ScalarValueOp::Xor: - case ScalarValueOp::ShiftLeft: - case ScalarValueOp::ShiftRight: - case ScalarValueOp::ShiftRightArithmetic: - case ScalarValueOp::BitFieldMaskU32: - case ScalarValueOp::BitFieldMaskU64Low: - case ScalarValueOp::BitFieldMaskU64High: return 2; - case ScalarValueOp::AddCarry: - case ScalarValueOp::Carry: - case ScalarValueOp::SubBorrow: - case ScalarValueOp::Borrow: - case ScalarValueOp::Add3: - case ScalarValueOp::ShiftLeftAdd: - case ScalarValueOp::ShiftLeftAddCarry: - case ScalarValueOp::AddShiftLeft: - case ScalarValueOp::XorAdd: - case ScalarValueOp::ShiftLeftOr: - case ScalarValueOp::ReadConst: return 3; - case ScalarValueOp::ReadConstBuffer: return 5; - default: return 0; - } -} - -namespace { - -constexpr uint32_t ScalarRegisters = 128; -constexpr uint32_t VectorRegisters = 256; -// Clamp X/Y/Z are consecutive three-bit fields; the high bit of each selects a border mode. -constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u); - -struct ScalarState { - std::array regs = {}; - std::array address_bases = {}; - uint32_t scc = ScalarProvenance::Undefined; - uint32_t m0 = ScalarProvenance::Undefined; - std::map vector_lanes; - - void Fill(uint32_t value) { - regs.fill(value); - address_bases.fill(value); - scc = value; - m0 = value; - vector_lanes.clear(); - } - - bool operator==(const ScalarState&) const = default; -}; - -struct ValueKey { - ScalarValueOp op = ScalarValueOp::Undefined; - uint32_t imm = 0; - std::array args = {}; - - auto operator<=>(const ValueKey&) const = default; -}; - -struct DescriptorKey { - std::array dwords = {}; - uint32_t dword_count = 0; - - auto operator<=>(const DescriptorKey&) const = default; -}; - -bool ScalarRegister(const Operand& operand, uint32_t& reg) { - if (operand.kind != OperandKind::Register) { - return false; - } - if (operand.reg.file == RegisterFile::Scalar && operand.reg.index < ScalarRegisters) { - reg = operand.reg.index; - return true; - } - if (operand.reg.file == RegisterFile::Vcc && operand.reg.index < 2) { - reg = 106u + operand.reg.index; - return true; - } - return false; -} - -bool VectorRegister(const Operand& operand, uint32_t& reg) { - if (operand.kind != OperandKind::Register || operand.reg.file != RegisterFile::Vector || - operand.reg.index >= VectorRegisters) { - return false; - } - reg = operand.reg.index; - return true; -} - -bool FlatStore(Opcode op) { - switch (op) { - case Opcode::FlatStoreByte: - case Opcode::FlatStoreShort: - case Opcode::FlatStoreDword: return true; - default: return false; - } -} - -bool PairDwordOpcode(Opcode op) { - switch (op) { - case Opcode::MoveU64: - case Opcode::WqmB64: - case Opcode::SaveexecB64: - case Opcode::BitwiseAndU64: - case Opcode::BitwiseAndNotU64: - case Opcode::BitwiseOrU64: - case Opcode::BitwiseOrNotU64: - case Opcode::BitwiseXorU64: - case Opcode::BitwiseNandU64: - case Opcode::BitwiseNorU64: - case Opcode::BitwiseXnorU64: - case Opcode::BitwiseNotU64: - case Opcode::BitFieldMaskU64: - case Opcode::BitFieldExtractU64: - case Opcode::BitReplicateB64B32: - case Opcode::ShiftLeftLogicalU64: - case Opcode::ShiftRightLogicalU64: - case Opcode::SelectU64: return true; - default: return false; - } -} - -bool ScalarMemoryLoad(Opcode op) { - return op == Opcode::SLoadDword || op == Opcode::SBufferLoadDword; -} - -bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector& visited) { - if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) { - return false; - } - if (visited[id] == 1) { - return false; - } - if (visited[id] == 2) { - return true; - } - if (visited[id] == 3) { - return false; - } - visited[id] = 1; - const auto& value = provenance.values[id]; - bool resolved = true; - if (value.op == ScalarValueOp::Phi) { - resolved = !value.phi_args.empty() && - std::all_of(value.phi_args.begin(), value.phi_args.end(), [&](uint32_t arg) { - return ValueResolved(provenance, arg, visited); - }); - } else { - const auto args = ScalarValueArgCount(value.op); - for (uint32_t i = 0; i < args && resolved; i++) { - resolved = ValueResolved(provenance, value.args[i], visited); - } - } - visited[id] = resolved ? 2 : 3; - return resolved; -} - -class Builder { -public: - explicit Builder(Program& program): m_program(program), m_graph(program.provenance) {} - - bool Run(std::string* error) { - m_graph = {}; - m_graph.values.resize(2); - m_graph.values[ScalarProvenance::Undefined].op = ScalarValueOp::Undefined; - m_graph.values[ScalarProvenance::Unknown].op = ScalarValueOp::Unknown; - - const auto block_count = m_program.blocks.size(); - m_entry.resize(block_count); - m_exit.resize(block_count); - m_phi.resize(block_count); - m_exit_ready.assign(block_count, false); - m_queued.assign(block_count, false); - for (auto& state: m_entry) { - state.Fill(ScalarProvenance::Undefined); - } - for (auto& state: m_exit) { - state.Fill(ScalarProvenance::Undefined); - } - for (auto& phis: m_phi) { - phis.Fill(ScalarProvenance::Undefined); - } - - m_user_data.Fill(ScalarProvenance::Undefined); - for (uint32_t reg = 0; reg < ScalarRegisters; reg++) { - m_user_data.regs[reg] = - reg >= m_program.user_data_base && - reg - m_program.user_data_base < m_program.user_data_count - ? InternValue({ScalarValueOp::UserData, 0, reg}) - : ScalarProvenance::Unknown; - } - m_user_data.scc = ScalarProvenance::Unknown; - m_user_data.m0 = ScalarProvenance::Unknown; - - if (block_count == 0) { - return true; - } - for (const auto& block: m_program.blocks) { - for (const auto predecessor: block.predecessors) { - if (predecessor >= block_count) { - return Fail(error, fmt::format("scalar provenance has invalid predecessor {}", - predecessor)); - } - } - for (const auto successor: block.successors) { - if (successor >= block_count) { - return Fail(error, fmt::format("scalar provenance has invalid successor {}", - successor)); - } - } - if (!ValidateScalarMemoryGroups(block, error)) { - return false; - } - } - Queue(0); - while (!m_work.empty()) { - const auto block_index = m_work.front(); - m_work.pop_front(); - m_queued[block_index] = false; - - auto next_entry = MergeEntry(block_index); - const bool entry_changed = next_entry != m_entry[block_index]; - m_entry[block_index] = next_entry; - - auto next_exit = Execute(m_program.blocks[block_index], next_entry, false); - const bool was_ready = m_exit_ready[block_index]; - if (was_ready && !entry_changed && next_exit == m_exit[block_index]) { - continue; - } - m_exit[block_index] = next_exit; - m_exit_ready[block_index] = true; - for (const auto successor: m_program.blocks[block_index].successors) { - Queue(successor); - } - } - - for (size_t block = 0; block < block_count; block++) { - Execute(m_program.blocks[block], m_entry[block], true); - } - return true; - } - -private: - bool Fail(std::string* error, std::string text) const { - if (error != nullptr) { - *error = std::move(text); - } - return false; - } - - uint32_t AddValue(ScalarValue value) { - m_graph.values.push_back(std::move(value)); - return static_cast(m_graph.values.size() - 1); - } - - bool ValidateScalarMemoryGroups(const BasicBlock& block, std::string* error) const { - for (size_t index = 0; index < block.instructions.size();) { - const auto& first = block.instructions[index]; - if (!ScalarMemoryLoad(first.op) || first.memory.component_count == 1) { - index++; - continue; - } - const auto count = first.memory.component_count; - uint32_t first_dst = 0; - if (count == 0 || first.memory.component_index != 0 || - index + count > block.instructions.size() || - !ScalarRegister(first.dst, first_dst)) { - return Fail(error, - fmt::format("malformed scalar-memory group at pc=0x{:x}", first.pc)); - } - for (uint32_t component = 0; component < count; component++) { - const auto& inst = block.instructions[index + component]; - uint32_t dst = 0; - if (inst.op != first.op || inst.pc != first.pc || - inst.memory.component_count != count || - inst.memory.component_index != component || - inst.memory.resource != first.memory.resource || - inst.memory.offset != first.memory.offset + component * 4u || - inst.src_count != first.src_count || - !std::equal(std::begin(inst.src), std::end(inst.src), std::begin(first.src)) || - !ScalarRegister(inst.dst, dst) || dst != first_dst + component) { - return Fail(error, - fmt::format("malformed scalar-memory component {} at pc=0x{:x}", - component, first.pc)); - } - } - index += count; - } - return true; - } - - uint32_t InternValue(ScalarValue value) { - const ValueKey key {value.op, value.imm, value.args}; - if (const auto found = m_values.find(key); found != m_values.end()) { - return found->second; - } - const auto id = AddValue(std::move(value)); - m_values.emplace(key, id); - return id; - } - - uint32_t Constant(uint32_t value) { return InternValue({ScalarValueOp::Constant, 0, value}); } - - static uint64_t VectorLaneKey(uint32_t reg, uint32_t lane) { - return (static_cast(reg) << 32u) | lane; - } - - uint32_t OperandValue(const Operand& operand, const ScalarState& state) { - uint32_t reg = 0; - if (ScalarRegister(operand, reg)) { - return state.regs[reg]; - } - if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::Scc) { - return state.scc; - } - if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::M0) { - return state.m0; - } - if (operand.kind == OperandKind::ImmediateU32 || - operand.kind == OperandKind::PcRelativeU32) { - if (operand.kind == OperandKind::PcRelativeU32) { - return InternValue({ScalarValueOp::PcRelativeLow, 0, operand.imm}); - } - return Constant(operand.imm); - } - return ScalarProvenance::Unknown; - } - - uint32_t Define(const Instruction& inst, ScalarValueOp op, const ScalarState& state, - uint32_t imm = 0) { - ScalarValue node; - node.op = op; - node.pc = inst.pc; - node.imm = imm; - for (uint32_t i = 0; i < inst.src_count && i < node.args.size(); i++) { - node.args[i] = OperandValue(inst.src[i], state); - } - return InternValue(std::move(node)); - } - - uint32_t DefineBorrow(const Instruction& inst, const ScalarState& state) { - ScalarValue node; - node.op = ScalarValueOp::Borrow; - node.pc = inst.pc; - node.args[0] = - inst.src_count > 0 ? OperandValue(inst.src[0], state) : ScalarProvenance::Unknown; - node.args[1] = - inst.src_count > 1 ? OperandValue(inst.src[1], state) : ScalarProvenance::Unknown; - node.args[2] = inst.src_count > 2 ? OperandValue(inst.src[2], state) : Constant(0); - return InternValue(std::move(node)); - } - - bool ConstantOperand(const Operand& operand, const ScalarState& state, uint32_t& value) { - const auto id = OperandValue(operand, state); - if (id >= m_graph.values.size() || m_graph.values[id].op != ScalarValueOp::Constant) { - return false; - } - value = m_graph.values[id].imm; - return true; - } - - uint32_t ReadVectorLane(const Instruction& inst, const ScalarState& state) { - if (inst.src_count < 2 || inst.src[0].kind != OperandKind::Register || - inst.src[0].reg.file != RegisterFile::Vector || - (m_program.wave_size != 32 && m_program.wave_size != 64)) { - return ScalarProvenance::Unknown; - } - uint32_t lane = 0; - if (!ConstantOperand(inst.src[1], state, lane)) { - return ScalarProvenance::Unknown; - } - const auto found = state.vector_lanes.find( - VectorLaneKey(inst.src[0].reg.index, lane % m_program.wave_size)); - return found != state.vector_lanes.end() ? found->second : ScalarProvenance::Unknown; - } - - void ClearVectorLanes(uint32_t reg, ScalarState& state) { - const auto first = state.vector_lanes.lower_bound(VectorLaneKey(reg, 0)); - const auto last = state.vector_lanes.lower_bound((static_cast(reg) + 1u) << 32u); - state.vector_lanes.erase(first, last); - } - - void WriteVectorDestination(const Instruction& inst, ScalarState& state) { - if (inst.dst.kind != OperandKind::Register || inst.dst.reg.file != RegisterFile::Vector) { - return; - } - const uint32_t reg = inst.dst.reg.index; - uint32_t address_base = ScalarProvenance::Unknown; - if (inst.op == Opcode::MoveU32 || inst.op == Opcode::IAddU32 || - inst.op == Opcode::IAddCarryU32 || inst.op == Opcode::ISubU32) { - for (uint32_t i = 0; i < inst.src_count; i++) { - uint32_t src_reg = 0; - if (VectorRegister(inst.src[i], src_reg) && - state.address_bases[src_reg] > ScalarProvenance::Unknown) { - address_base = state.address_bases[src_reg]; - break; - } - } - if (address_base == ScalarProvenance::Unknown) { - for (uint32_t i = 0; i < inst.src_count; i++) { - uint32_t src_reg = 0; - if (ScalarRegister(inst.src[i], src_reg) && - state.regs[src_reg] > ScalarProvenance::Unknown) { - address_base = state.regs[src_reg]; - break; - } - } - } - } - state.address_bases[reg] = ScalarProvenance::Unknown; - if (inst.op == Opcode::MoveRelDestU32) { - state.vector_lanes.clear(); - return; - } - if (inst.op != Opcode::WriteLaneU32) { - uint32_t dwords = std::max(inst.memory.data_dwords, 1u); - if (PairDwordOpcode(inst.op) || inst.op == Opcode::UMadU64U32) { - dwords = std::max(dwords, 2u); - } - for (uint32_t i = 0; i < dwords && reg <= UINT32_MAX - i; i++) { - ClearVectorLanes(reg + i, state); - if (reg + i < state.address_bases.size()) { - state.address_bases[reg + i] = ScalarProvenance::Unknown; - } - } - state.address_bases[reg] = address_base; - return; - } - if (inst.src_count < 2 || (m_program.wave_size != 32 && m_program.wave_size != 64)) { - ClearVectorLanes(reg, state); - return; - } - uint32_t lane = 0; - if (!ConstantOperand(inst.src[1], state, lane)) { - ClearVectorLanes(reg, state); - return; - } - state.vector_lanes[VectorLaneKey(reg, lane % m_program.wave_size)] = - OperandValue(inst.src[0], state); - } - - uint32_t ReadConst(const Instruction& inst, const ScalarState& state, bool buffer) { - ScalarValue node; - node.op = buffer ? ScalarValueOp::ReadConstBuffer : ScalarValueOp::ReadConst; - node.pc = inst.pc; - const auto base = buffer ? inst.memory.resource * 4u : inst.memory.resource; - if (base + 1u >= ScalarRegisters) { - return ScalarProvenance::Unknown; - } - node.imm = inst.memory.offset; - if (buffer) { - if (base + 3u >= ScalarRegisters) { - return ScalarProvenance::Unknown; - } - for (uint32_t i = 0; i < 4; i++) { - node.args[i] = state.regs[base + i]; - } - node.args[4] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0); - } else { - node.args[0] = state.regs[base]; - node.args[1] = state.regs[base + 1u]; - node.args[2] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0); - } - return InternValue(std::move(node)); - } - - ScalarValueOp Operation(Opcode op) const { - switch (op) { - case Opcode::IAddU32: - case Opcode::ScalarSignedAddOverflowI32: return ScalarValueOp::Add; - case Opcode::IAddCarryU32: - case Opcode::ScalarAddCarryU32: return ScalarValueOp::AddCarry; - case Opcode::ISubU32: - case Opcode::ScalarSubBorrowU32: - case Opcode::ScalarSignedSubOverflowI32: return ScalarValueOp::Sub; - case Opcode::ScalarSubBorrowCarryU32: return ScalarValueOp::SubBorrow; - case Opcode::IMulU32: return ScalarValueOp::Mul; - case Opcode::BitwiseAndU32: return ScalarValueOp::And; - case Opcode::BitwiseAndNotU32: return ScalarValueOp::AndNot; - case Opcode::BitwiseOrU32: return ScalarValueOp::Or; - case Opcode::BitwiseOrNotU32: return ScalarValueOp::OrNot; - case Opcode::BitwiseXorU32: return ScalarValueOp::Xor; - case Opcode::BitwiseNotU32: return ScalarValueOp::Not; - case Opcode::ShiftLeftLogicalU32: return ScalarValueOp::ShiftLeft; - case Opcode::ShiftRightLogicalU32: return ScalarValueOp::ShiftRight; - case Opcode::ShiftRightArithmeticI32: return ScalarValueOp::ShiftRightArithmetic; - case Opcode::BitFieldMaskU32: return ScalarValueOp::BitFieldMaskU32; - case Opcode::IAdd3U32: return ScalarValueOp::Add3; - case Opcode::ScalarShiftLeftAddCarryU32: - case Opcode::ShiftLeftAddU32: return ScalarValueOp::ShiftLeftAdd; - case Opcode::AddShiftLeftU32: return ScalarValueOp::AddShiftLeft; - case Opcode::XorAddU32: return ScalarValueOp::XorAdd; - case Opcode::ShiftLeftOrU32: return ScalarValueOp::ShiftLeftOr; - default: return ScalarValueOp::Unknown; - } - } - - void UpdateScc(const Instruction& inst, const ScalarState& before, ScalarState& state) { - switch (inst.op) { - case Opcode::ScalarAddCarryU32: - state.scc = Define(inst, ScalarValueOp::Carry, before); - break; - case Opcode::ScalarSubBorrowU32: - case Opcode::ScalarSubBorrowCarryU32: state.scc = DefineBorrow(inst, before); break; - case Opcode::ScalarShiftLeftAddCarryU32: - state.scc = Define(inst, ScalarValueOp::ShiftLeftAddCarry, before); - break; - default: break; - } - } - - void WriteDestination(const Instruction& inst, ScalarState& state) { - if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Scc) { - state.scc = inst.op == Opcode::MoveU32 && inst.src_count != 0 - ? OperandValue(inst.src[0], state) - : ScalarProvenance::Unknown; - return; - } - if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::M0) { - const auto before = state; - const auto op = Operation(inst.op); - state.m0 = ScalarProvenance::Unknown; - if (inst.op == Opcode::MoveU32 && inst.src_count != 0) { - state.m0 = OperandValue(inst.src[0], before); - } else if (op != ScalarValueOp::Unknown) { - state.m0 = Define(inst, op, before); - } - UpdateScc(inst, before, state); - return; - } - - uint32_t dst = 0; - if (!ScalarRegister(inst.dst, dst)) { - return; - } - - const auto before = state; - state.regs[dst] = ScalarProvenance::Unknown; - if (PairDwordOpcode(inst.op) && dst + 1 < ScalarRegisters) { - state.regs[dst + 1] = ScalarProvenance::Unknown; - } - - uint32_t value = ScalarProvenance::Unknown; - switch (inst.op) { - case Opcode::MoveU32: - case Opcode::MoveF32Bits: - if (inst.src_count != 0) { - if (inst.src[0].kind == OperandKind::PcRelativeU32) { - value = - InternValue({ScalarValueOp::PcRelativeLow, inst.pc, inst.src[0].imm}); - } else { - value = OperandValue(inst.src[0], before); - } - } - if (inst.op == Opcode::MoveU32 && inst.src_count != 0 && - inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].imm == 0 && - dst != 0) { - const auto low = before.regs[dst - 1]; - if (low < m_graph.values.size() && - m_graph.values[low].op == ScalarValueOp::PcRelativeLow && - m_graph.values[low].pc == inst.pc) { - value = InternValue( - {ScalarValueOp::PcRelativeHigh, inst.pc, m_graph.values[low].imm}); - } - } - break; - case Opcode::MoveU64: { - uint32_t src = 0; - if (inst.src_count != 0 && dst + 1 < ScalarRegisters) { - if (ScalarRegister(inst.src[0], src) && src + 1 < ScalarRegisters) { - value = before.regs[src]; - state.regs[dst + 1] = before.regs[src + 1]; - } else { - value = OperandValue(inst.src[0], before); - state.regs[dst + 1] = Constant( - inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].sext_64 - ? UINT32_MAX - : 0u); - } - } - break; - } - case Opcode::BitFieldMaskU64: - value = Define(inst, ScalarValueOp::BitFieldMaskU64Low, before); - if (dst + 1 < ScalarRegisters) { - state.regs[dst + 1] = Define(inst, ScalarValueOp::BitFieldMaskU64High, before); - } - break; - case Opcode::SLoadDword: value = ReadConst(inst, before, false); break; - case Opcode::SBufferLoadDword: value = ReadConst(inst, before, true); break; - case Opcode::ReadLaneU32: value = ReadVectorLane(inst, before); break; - default: { - const auto op = Operation(inst.op); - if (op != ScalarValueOp::Unknown) { - value = Define(inst, op, before); - } - break; - } - } - state.regs[dst] = value; - UpdateScc(inst, before, state); - - uint32_t dst2 = 0; - if (ScalarRegister(inst.dst2, dst2)) { - state.regs[dst2] = - inst.src_count > 1 ? OperandValue(inst.src[1], before) : ScalarProvenance::Unknown; - } - } - - uint32_t AddDescriptor(DescriptorValue descriptor) { - const DescriptorKey key {descriptor.dwords, descriptor.dword_count}; - if (const auto found = m_descriptors.find(key); found != m_descriptors.end()) { - return found->second; - } - m_graph.descriptors.push_back(descriptor); - const auto id = static_cast(m_graph.descriptors.size() - 1) + 2u; - m_descriptors.emplace(key, id); - return id; - } - - uint32_t AddDescriptor(const ScalarState& state, uint32_t base, uint32_t dwords) { - if (base >= ScalarRegisters || dwords == 0 || dwords > 8 || - base + dwords > ScalarRegisters) { - return ScalarProvenance::Unknown; - } - DescriptorValue descriptor; - descriptor.dword_count = dwords; - for (uint32_t i = 0; i < dwords; i++) { - descriptor.dwords[i] = state.regs[base + i]; - } - return AddDescriptor(descriptor); - } - - uint32_t AddSamplerDescriptor(const ScalarState& state, uint32_t base) { - if (base >= ScalarRegisters || 4u > ScalarRegisters - base) { - return ScalarProvenance::Unknown; - } - DescriptorValue descriptor; - descriptor.dword_count = 4; - for (uint32_t i = 0; i < 4; i++) { - descriptor.dwords[i] = state.regs[base + i]; - } - if (auto d0 = descriptor.dwords[0]; - d0 < m_graph.values.size() && m_graph.values[d0].op == ScalarValueOp::Constant && - (m_graph.values[d0].imm & SamplerBorderClampMask) == 0) { - // Without a border clamp, the border color and table index in dword 3 are unused. - descriptor.dwords[3] = Constant(0); - } - return AddDescriptor(descriptor); - } - - uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) { - const uint32_t first = FlatStore(inst.op) ? 1u : 0u; - if (inst.src_count < first + 2u) { - return ScalarProvenance::Unknown; - } - uint32_t low = 0; - uint32_t high = 0; - if (!VectorRegister(inst.src[first], low) || !VectorRegister(inst.src[first + 1u], high) || - state.address_bases[low] <= ScalarProvenance::Unknown || - state.address_bases[high] <= ScalarProvenance::Unknown) { - return ScalarProvenance::Unknown; - } - DescriptorValue descriptor; - descriptor.dword_count = 2; - descriptor.dwords[0] = state.address_bases[low]; - descriptor.dwords[1] = state.address_bases[high]; - return AddDescriptor(descriptor); - } - - void AttachSources(Instruction& inst, const ScalarState& state) { - inst.memory.resource_source = 0; - inst.memory.sampler_source = 0; - if (inst.op == Opcode::SLoadDword) { - inst.memory.resource_source = AddDescriptor(state, inst.memory.resource, 2); - return; - } - if (inst.memory.kind == ResourceKind::Flat || inst.memory.kind == ResourceKind::Global || - inst.memory.kind == ResourceKind::Scratch) { - inst.memory.resource_source = ScalarProvenance::Unknown; - if (inst.memory.kind == ResourceKind::Flat) { - inst.memory.resource_source = AddFlatAddressDescriptor(inst, state); - } - for (uint32_t i = 0; i < inst.src_count; i++) { - const auto& source = inst.src[i]; - if (inst.memory.resource_source == ScalarProvenance::Unknown && - source.kind == OperandKind::Register && - source.reg.file == RegisterFile::Scalar) { - inst.memory.resource_source = AddDescriptor(state, source.reg.index, 2); - break; - } - } - return; - } - if (inst.memory.kind == ResourceKind::Buffer || - (inst.memory.kind == ResourceKind::ScalarBuffer && - inst.op == Opcode::SBufferLoadDword)) { - const auto base = inst.memory.resource * 4u; - inst.memory.resource_source = AddDescriptor(state, base, 4); - return; - } - if (inst.memory.kind == ResourceKind::Image || - inst.memory.kind == ResourceKind::ImageUint || - inst.memory.kind == ResourceKind::StorageImage || - inst.memory.kind == ResourceKind::StorageImageUint) { - inst.memory.resource_source = AddDescriptor(state, inst.memory.resource * 4u, 8); - if (inst.op == Opcode::ImageSample || inst.op == Opcode::ImageGather4 || - inst.op == Opcode::ImageGetLod) { - inst.memory.sampler_source = AddSamplerDescriptor(state, inst.memory.sampler * 4u); - } - } - } - - void AttachScalarSources(Instruction& inst, const ScalarState& state) { - std::fill(std::begin(inst.scalar_sources), std::end(inst.scalar_sources), - ScalarProvenance::Undefined); - for (uint32_t i = 0; i < inst.src_count; i++) { - const auto& source = inst.src[i]; - if (source.kind == OperandKind::Register && source.reg.file != RegisterFile::Vector && - source.reg.file != RegisterFile::Exec) { - inst.scalar_sources[i] = OperandValue(source, state); - } - } - } - - ScalarState Execute(BasicBlock& block, ScalarState state, bool attach_sources) { - for (size_t index = 0; index < block.instructions.size();) { - auto& inst = block.instructions[index]; - uint32_t first_dst = 0; - if (ScalarMemoryLoad(inst.op) && inst.memory.component_count > 1 && - ScalarRegister(inst.dst, first_dst)) { - const auto before = state; - for (uint32_t component = 0; component < inst.memory.component_count; component++) { - auto& component_inst = block.instructions[index + component]; - if (attach_sources) { - component_inst.scalar_value = ScalarProvenance::Undefined; - AttachScalarSources(component_inst, before); - AttachSources(component_inst, before); - } - auto component_state = before; - WriteDestination(component_inst, component_state); - uint32_t dst = 0; - ScalarRegister(component_inst.dst, dst); - state.regs[dst] = component_state.regs[dst]; - if (attach_sources) { - component_inst.scalar_value = component_state.regs[dst]; - } - } - index += inst.memory.component_count; - continue; - } - if (attach_sources) { - inst.scalar_value = ScalarProvenance::Undefined; - AttachScalarSources(inst, state); - AttachSources(inst, state); - } - WriteVectorDestination(inst, state); - WriteDestination(inst, state); - if (attach_sources) { - uint32_t dst = 0; - if (ScalarRegister(inst.dst, dst)) { - inst.scalar_value = state.regs[dst]; - } - } - index++; - } - return state; - } - - ScalarState MergeEntry(size_t block_index) { - ScalarState result; - result.Fill(ScalarProvenance::Undefined); - const auto& block = m_program.blocks[block_index]; - const auto merge = [&](uint32_t initial, auto predecessor_value, uint32_t* phi) { - std::vector incoming; - if (initial != ScalarProvenance::Undefined) { - incoming.push_back(initial); - } - for (const auto predecessor: block.predecessors) { - const auto value = predecessor_value(predecessor, m_exit[predecessor]); - if (value != ScalarProvenance::Undefined && - std::find(incoming.begin(), incoming.end(), value) == incoming.end()) { - incoming.push_back(value); - } - } - if (incoming.empty()) { - return ScalarProvenance::Undefined; - } - if (*phi == ScalarProvenance::Undefined) { - if (incoming.size() == 1) { - return incoming[0]; - } - *phi = AddValue({ScalarValueOp::Phi, block.start_pc}); - } - m_graph.values[*phi].phi_args = std::move(incoming); - return *phi; - }; - - for (uint32_t reg = 0; reg < ScalarRegisters; reg++) { - const auto initial = - block_index == 0 ? m_user_data.regs[reg] : ScalarProvenance::Undefined; - result.regs[reg] = merge( - initial, [reg](uint32_t, const ScalarState& state) { return state.regs[reg]; }, - &m_phi[block_index].regs[reg]); - } - for (uint32_t reg = 0; reg < VectorRegisters; reg++) { - result.address_bases[reg] = merge( - block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined, - [reg](uint32_t, const ScalarState& state) { return state.address_bases[reg]; }, - &m_phi[block_index].address_bases[reg]); - } - result.scc = merge( - block_index == 0 ? m_user_data.scc : ScalarProvenance::Undefined, - [](uint32_t, const ScalarState& state) { return state.scc; }, &m_phi[block_index].scc); - result.m0 = merge( - block_index == 0 ? m_user_data.m0 : ScalarProvenance::Undefined, - [](uint32_t, const ScalarState& state) { return state.m0; }, &m_phi[block_index].m0); - - std::vector vector_lanes; - for (const auto predecessor: block.predecessors) { - for (const auto& [lane, value]: m_exit[predecessor].vector_lanes) { - (void)value; - vector_lanes.push_back(lane); - } - } - std::sort(vector_lanes.begin(), vector_lanes.end()); - vector_lanes.erase(std::unique(vector_lanes.begin(), vector_lanes.end()), - vector_lanes.end()); - for (const auto lane: vector_lanes) { - auto& phi = m_phi[block_index].vector_lanes[lane]; - result.vector_lanes[lane] = merge( - block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined, - [this, lane](uint32_t predecessor, const ScalarState& state) { - if (!m_exit_ready[predecessor]) { - return ScalarProvenance::Undefined; - } - const auto found = state.vector_lanes.find(lane); - return found != state.vector_lanes.end() ? found->second - : ScalarProvenance::Unknown; - }, - &phi); - } - return result; - } - - void Queue(uint32_t block) { - if (!m_queued[block]) { - m_queued[block] = true; - m_work.push_back(block); - } - } - - Program& m_program; - ScalarProvenance& m_graph; - ScalarState m_user_data; - std::vector m_entry; - std::vector m_exit; - std::vector m_phi; - std::vector m_exit_ready; - std::vector m_queued; - std::deque m_work; - std::map m_values; - std::map m_descriptors; -}; - -} // namespace - -bool BuildScalarProvenance(Program& program, std::string* error) { - if (program.resource_tracking_complete) { - if (error != nullptr) { - *error = "cannot rebuild scalar provenance after resource tracking"; - } - return false; - } - if (program.user_data_count > 64) { - if (error != nullptr) { - *error = "scalar provenance user-data count exceeds 64 SGPRs"; - } - return false; - } - if (program.user_data_base > ScalarRegisters || - program.user_data_count > ScalarRegisters - program.user_data_base) { - if (error != nullptr) { - *error = "scalar provenance user-data window is out of range"; - } - return false; - } - if (program.srt_patching_complete) { - if (error != nullptr) { - *error = "cannot rebuild scalar provenance after SRT patching"; - } - return false; - } - program.srt = {}; - program.srt_plan_complete = false; - return Builder(program).Run(error); -} - -const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source) { - if (source < 2 || source - 2 >= program.provenance.descriptors.size()) { - return nullptr; - } - return &program.provenance.descriptors[source - 2]; -} - -bool DescriptorSourceResolved(const Program& program, uint32_t source) { - const auto* descriptor = GetDescriptorSource(program, source); - if (descriptor == nullptr) { - return false; - } - std::vector visited(program.provenance.values.size()); - for (uint32_t i = 0; i < descriptor->dword_count; i++) { - if (!ValueResolved(program.provenance, descriptor->dwords[i], visited)) { - return false; - } - } - return true; -} - -std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value) { - if (value >= provenance.values.size()) { - return "invalid"; - } - const auto& node = provenance.values[value]; - switch (node.op) { - case ScalarValueOp::Undefined: return "undefined"; - case ScalarValueOp::Unknown: return "unknown"; - case ScalarValueOp::UserData: return fmt::format("ud{}", node.imm); - case ScalarValueOp::Constant: return fmt::format("0x{:08x}", node.imm); - case ScalarValueOp::PcRelativeLow: return fmt::format("pc_lo+0x{:x}", node.imm); - case ScalarValueOp::PcRelativeHigh: return fmt::format("pc_hi+0x{:x}", node.imm); - case ScalarValueOp::ReadConst: - return fmt::format("read_const({}, {}, {}, +{})", node.args[0], node.args[1], - node.args[2], node.imm); - case ScalarValueOp::ReadConstBuffer: - return fmt::format("read_const_buffer({}, {}, {}, {}, {}, +{})", node.args[0], - node.args[1], node.args[2], node.args[3], node.args[4], node.imm); - case ScalarValueOp::Phi: return fmt::format("phi{}", value); - case ScalarValueOp::BitFieldMaskU32: - return fmt::format("bfm_u32({}, {})", node.args[0], node.args[1]); - default: return fmt::format("value{}", value); - } -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ScalarProvenance.h b/src/graphics/shader/recompiler/ir/ScalarProvenance.h deleted file mode 100644 index 5597e1653..000000000 --- a/src/graphics/shader/recompiler/ir/ScalarProvenance.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -namespace Libs::Graphics::ShaderRecompiler::IR { - -// Builds scalar reaching definitions and attaches descriptor sources to individual memory uses. -bool BuildScalarProvenance(Program& program, std::string* error); - -uint32_t ScalarValueArgCount(ScalarValueOp op); - -const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source); -bool DescriptorSourceResolved(const Program& program, uint32_t source); - -std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value); - -} // namespace Libs::Graphics::ShaderRecompiler::IR - -#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ */ diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.cpp b/src/graphics/shader/recompiler/ir/ShaderIR.cpp index c888f7c91..5adf2da81 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIR.cpp @@ -32,6 +32,13 @@ Operand MakePcRelativeU32(uint32_t value) { return operand; } +Operand MakePcRelativeHighU32(uint32_t value) { + Operand operand; + operand.kind = OperandKind::PcRelativeHighU32; + operand.imm = value; + return operand; +} + bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { if (!ScalarResultWritesSccNonZero(decoded.opcode)) { @@ -74,6 +81,7 @@ uint32_t VectorByteConvertIndex(Decoder::Opcode opcode) { void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand) { operand.sdwa_sel = decoded.sdwa_sel; + operand.sdwa_dst_unused = decoded.sdwa_dst_unused; operand.omod = decoded.omod; operand.sdwa_sext = decoded.sdwa_sext; operand.op_sel = decoded.op_sel; @@ -214,17 +222,20 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi mem.image_nsa_addr[i] = decoded.image_nsa_addr[i]; } mem.memory_segment = decoded.memory_segment; - mem.data_signed = decoded.data_signed; - mem.typed = decoded.typed; - mem.formatted = decoded.formatted; - mem.image_has_mip = decoded.opcode == Decoder::Opcode::ImageLoadMip || - decoded.opcode == Decoder::Opcode::ImageStoreMip; - mem.glc = decoded.glc; - mem.slc = decoded.slc; - mem.idxen = decoded.idxen; - mem.offen = decoded.offen; - mem.resource = ResourceIndexFromOperand(decoded.src1); - mem.sampler = ResourceIndexFromOperand(decoded.src2); + mem.address_is_full = + kind == ResourceKind::Flat || kind == ResourceKind::Scratch || + (kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr); + mem.data_signed = decoded.data_signed; + mem.typed = decoded.typed; + mem.formatted = decoded.formatted; + mem.image_has_mip = decoded.opcode == Decoder::Opcode::ImageLoadMip || + decoded.opcode == Decoder::Opcode::ImageStoreMip; + mem.glc = decoded.glc; + mem.slc = decoded.slc; + mem.idxen = decoded.idxen; + mem.offen = decoded.offen; + mem.resource = ResourceIndexFromOperand(decoded.src1); + mem.sampler = ResourceIndexFromOperand(decoded.src2); if (kind == ResourceKind::Lds || kind == ResourceKind::Gds) { mem.resource = 0; mem.sampler = 0; @@ -246,6 +257,7 @@ uint32_t RawScalarLoadBase(const Decoder::Operand& operand) { void ClearRegisterOffsetModifiers(Decoder::Operand& operand) { operand.sdwa_sel = 6; + operand.sdwa_dst_unused = 2; operand.omod = 0; operand.sdwa_sext = false; operand.op_sel = false; @@ -988,9 +1000,10 @@ bool LowerScalarGetpcB64(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) { for (uint32_t i = 0; i < 2u; i++) { Instruction inst; - inst.pc = decoded.pc; - inst.op = Opcode::MoveU32; - inst.src[0] = i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakeImmediateU32(0u); + inst.pc = decoded.pc; + inst.op = Opcode::MoveU32; + inst.src[0] = + i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakePcRelativeHighU32(decoded.pc + 4u); inst.src_count = 1; if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error)) { return false; diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 8eb3763bd..c9f4bce5b 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -4,52 +4,19 @@ #include "common/common.h" #include "common/stringUtils.h" #include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/shader/recompiler/cfg/ShaderCFG.h" -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" +#include "graphics/shader/recompiler/ir/opcodes/Opcodes.h" #include "graphics/shader/shader.h" #include +#include #include #include namespace Libs::Graphics::ShaderRecompiler::IR { -enum class Opcode { -#define IR_OPCODE(name, category) name, -#include "graphics/shader/recompiler/ir/ShaderIROpcodes.inc" -#undef IR_OPCODE - Count, -}; - -enum class OpcodeClass { General, Compare, Compare64 }; - -struct OpcodeInfo { - std::string_view name; - OpcodeClass opcode_class = OpcodeClass::General; -}; - -inline constexpr OpcodeInfo OPCODE_INFO[] = { -#define IR_OPCODE(name, category) {#name, OpcodeClass::category}, -#include "graphics/shader/recompiler/ir/ShaderIROpcodes.inc" -#undef IR_OPCODE -}; -static_assert(sizeof(OPCODE_INFO) / sizeof(OPCODE_INFO[0]) == static_cast(Opcode::Count)); - -constexpr const OpcodeInfo& GetOpcodeInfo(Opcode opcode) { - return OPCODE_INFO[static_cast(opcode)]; -} - -constexpr std::string_view OpcodeName(Opcode opcode) { - return GetOpcodeInfo(opcode).name; -} - -constexpr bool IsCompareOpcode(Opcode opcode) { - return GetOpcodeInfo(opcode).opcode_class != OpcodeClass::General; -} - -constexpr bool IsCompare64Opcode(Opcode opcode) { - return GetOpcodeInfo(opcode).opcode_class == OpcodeClass::Compare64; -} +struct ValueProgram; enum class RegisterFile { Scalar, Vector, Vcc, Exec, Scc, M0 }; @@ -62,7 +29,7 @@ struct Register { } }; -enum class OperandKind { Register, ImmediateU32, PcRelativeU32, Null }; +enum class OperandKind { Register, ImmediateU32, PcRelativeU32, PcRelativeHighU32, Null }; struct Operand { OperandKind kind = OperandKind::Null; @@ -70,6 +37,7 @@ struct Operand { uint32_t imm = 0; bool sext_64 = false; uint32_t sdwa_sel = 6; + uint32_t sdwa_dst_unused = 2; uint32_t omod = 0; bool sdwa_sext = false; bool op_sel = false; @@ -123,19 +91,17 @@ struct MemoryInfo { uint32_t image_nsa_dwords = 0; uint32_t image_nsa_addr[Decoder::MaxImageNsaAddressComponents] = {}; uint32_t memory_segment = 0; - // Dense resource indices are assigned by resource tracking. These source IDs refer to the - // exact scalar definitions reaching this instruction before that patching step. - uint32_t resource_source = 0; - uint32_t sampler_source = 0; - bool data_signed = false; - bool typed = false; - bool formatted = false; - bool image_has_mip = false; - bool image_cube = false; - bool glc = false; - bool slc = false; - bool idxen = false; - bool offen = false; + bool address_is_full = false; + bool data_signed = false; + bool typed = false; + bool formatted = false; + bool image_has_mip = false; + bool image_cube = false; + bool glc = false; + bool slc = false; + bool idxen = false; + bool offen = false; + bool planning_only = false; bool operator==(const MemoryInfo& other) const = default; }; @@ -169,9 +135,7 @@ struct Instruction { Operand dst; Operand dst2; Operand src[4]; - uint32_t src_count = 0; - uint32_t scalar_sources[4] = {}; - uint32_t scalar_value = 0; + uint32_t src_count = 0; MemoryInfo memory; ExportInfo export_info; InputInfo input_info; @@ -192,53 +156,6 @@ struct BasicBlock { std::vector instructions; }; -enum class ScalarValueOp { - Undefined, - Unknown, - UserData, - Constant, - PcRelativeLow, - PcRelativeHigh, - Add, - AddCarry, - Carry, - Sub, - SubBorrow, - Borrow, - Mul, - And, - AndNot, - Or, - OrNot, - Xor, - Not, - ShiftLeft, - ShiftRight, - ShiftRightArithmetic, - BitFieldMaskU32, - BitFieldMaskU64Low, - BitFieldMaskU64High, - Add3, - ShiftLeftAdd, - ShiftLeftAddCarry, - AddShiftLeft, - XorAdd, - ShiftLeftOr, - ReadConst, - ReadConstBuffer, - Phi, -}; - -struct ScalarValue { - ScalarValueOp op = ScalarValueOp::Undefined; - uint32_t pc = 0; - uint32_t imm = 0; - std::array args = {}; - std::vector phi_args; - - bool operator==(const ScalarValue& other) const = default; -}; - struct DescriptorValue { std::array dwords = {}; uint32_t dword_count = 0; @@ -248,32 +165,6 @@ struct DescriptorValue { } }; -struct ScalarProvenance { - static constexpr uint32_t Undefined = 0; - static constexpr uint32_t Unknown = 1; - - std::vector values; - std::vector descriptors; - - bool operator==(const ScalarProvenance& other) const = default; -}; - -struct SrtRead { - uint32_t value = 0; - uint32_t flat_offset = 0; - uint32_t use_pc = 0; - - bool operator==(const SrtRead& other) const = default; -}; - -struct SrtPlan { - std::vector reads; - std::vector dynamic_reads; - std::vector dynamic_sources; - - bool operator==(const SrtPlan& other) const = default; -}; - struct BufferResource { static constexpr uint32_t NoImageAlias = UINT32_MAX; @@ -328,11 +219,12 @@ struct SampledResourcePair { }; struct AddressResource { - uint32_t source = 0; + uint32_t source = UINT32_MAX; uint32_t first_use_pc = 0; ResourceKind kind = ResourceKind::Flat; int32_t min_offset = 0; uint64_t specialized_base = 0; + bool unbased = true; bool read = false; bool written = false; bool atomic = false; @@ -451,25 +343,23 @@ struct ShaderInfo { }; struct Program { - ShaderType stage = ShaderType::Unknown; - ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; - uint64_t shader_hash = 0; - uint32_t wave_size = 64; - uint32_t user_data_base = 0; - uint32_t user_data_count = 64; - bool dispatcher_fallback = false; - CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; - std::string fallback_reason; - std::vector blocks; - ScalarProvenance provenance; - SrtPlan srt; - bool srt_plan_complete = false; - bool srt_patching_complete = false; - ShaderInfo info; - bool resource_tracking_complete = false; - bool shader_info_complete = false; - BindingLayout bindings; - bool binding_layout_complete = false; + ShaderType stage = ShaderType::Unknown; + ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave; + uint64_t shader_hash = 0; + uint32_t wave_size = 64; + uint32_t user_data_base = 0; + uint32_t user_data_count = 64; + bool dispatcher_fallback = false; + CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; + std::string fallback_reason; + std::vector blocks; + std::shared_ptr values; + bool srt_plan_complete = false; + ShaderInfo info; + bool resource_tracking_complete = false; + bool shader_info_complete = false; + BindingLayout bindings; + bool binding_layout_complete = false; }; bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage, diff --git a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h index f94b58e10..0bb0cb69a 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRInternal.h +++ b/src/graphics/shader/recompiler/ir/ShaderIRInternal.h @@ -18,6 +18,7 @@ bool ScalarResultIs64Bit(Decoder::Opcode opcode); Operand MakeSccOperand(); Operand MakeImmediateU32(uint32_t value); Operand MakePcRelativeU32(uint32_t value); +Operand MakePcRelativeHighU32(uint32_t value); bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error); uint32_t VectorByteConvertIndex(Decoder::Opcode opcode); diff --git a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp index cda89931f..1be377192 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp +++ b/src/graphics/shader/recompiler/ir/ShaderIRLog.cpp @@ -39,6 +39,9 @@ std::string OperandToString(const Operand& operand) { case OperandKind::Register: text = RegisterToString(operand.reg); break; case OperandKind::ImmediateU32: text = fmt::format("0x{:08x}", operand.imm); break; case OperandKind::PcRelativeU32: text = fmt::format("pc_rel(0x{:08x})", operand.imm); break; + case OperandKind::PcRelativeHighU32: + text = fmt::format("pc_rel_hi(0x{:08x})", operand.imm); + break; case OperandKind::Null: text = "null"; break; default: text = "unknown"; break; } diff --git a/src/graphics/shader/recompiler/ir/ShaderIROpcodes.inc b/src/graphics/shader/recompiler/ir/ShaderIROpcodes.inc deleted file mode 100644 index 4996873a6..000000000 --- a/src/graphics/shader/recompiler/ir/ShaderIROpcodes.inc +++ /dev/null @@ -1,342 +0,0 @@ -// Central IR opcode catalog. Keep semantic categories here so enum identity, logging, -// and generic analysis cannot drift apart. -IR_OPCODE(ControlNop, General) -IR_OPCODE(Waitcnt, General) -IR_OPCODE(Barrier, General) -IR_OPCODE(Sendmsg, General) -IR_OPCODE(TtraceData, General) -IR_OPCODE(InstPrefetch, General) -IR_OPCODE(MoveU32, General) -IR_OPCODE(MoveF32Bits, General) -IR_OPCODE(MoveRelDestU32, General) -IR_OPCODE(MoveRelSourceU32, General) -IR_OPCODE(MoveU64, General) -IR_OPCODE(WqmB64, General) -IR_OPCODE(SaveexecB32, General) -IR_OPCODE(SaveexecB64, General) -IR_OPCODE(ReadFirstLaneU32, General) -IR_OPCODE(ReadLaneU32, General) -IR_OPCODE(WriteLaneU32, General) -IR_OPCODE(Permlane16B32, General) -IR_OPCODE(Permlanex16B32, General) -IR_OPCODE(AbsI32, General) -IR_OPCODE(IAddU32, General) -IR_OPCODE(IAddCarryU32, General) -IR_OPCODE(ISubBorrowU32, General) -IR_OPCODE(ScalarAddCarryU32, General) -IR_OPCODE(ScalarSubBorrowU32, General) -IR_OPCODE(ScalarSubBorrowCarryU32, General) -IR_OPCODE(ScalarSignedAddOverflowI32, General) -IR_OPCODE(ScalarSignedSubOverflowI32, General) -IR_OPCODE(ScalarShiftLeftAddCarryU32, General) -IR_OPCODE(ISubU32, General) -IR_OPCODE(IMulU32, General) -IR_OPCODE(UMulHighU32, General) -IR_OPCODE(SMulHighI32, General) -IR_OPCODE(IMadI24U32, General) -IR_OPCODE(UMadU24U32, General) -IR_OPCODE(UMadU64U32, General) -IR_OPCODE(SadU32, General) -IR_OPCODE(IAdd3U32, General) -IR_OPCODE(IMulI24U32, General) -IR_OPCODE(UMulU24U32, General) -IR_OPCODE(IMinI32, General) -IR_OPCODE(IMaxI32, General) -IR_OPCODE(IMin3I32, General) -IR_OPCODE(IMax3I32, General) -IR_OPCODE(IMed3I32, General) -IR_OPCODE(UMinU32, General) -IR_OPCODE(UMaxU32, General) -IR_OPCODE(UMin3U32, General) -IR_OPCODE(UMax3U32, General) -IR_OPCODE(UMed3U32, General) -IR_OPCODE(BitwiseAndU32, General) -IR_OPCODE(BitwiseAndU64, General) -IR_OPCODE(BitwiseAndNotU32, General) -IR_OPCODE(BitwiseAndNotU64, General) -IR_OPCODE(BitwiseOrU32, General) -IR_OPCODE(BitwiseOrU64, General) -IR_OPCODE(BitwiseOrNotU32, General) -IR_OPCODE(BitwiseOrNotU64, General) -IR_OPCODE(BitwiseAndOrU32, General) -IR_OPCODE(BitwiseOr3U32, General) -IR_OPCODE(BitwiseXorU32, General) -IR_OPCODE(BitwiseXorU64, General) -IR_OPCODE(BitwiseXor3U32, General) -IR_OPCODE(BitwiseNandU32, General) -IR_OPCODE(BitwiseNandU64, General) -IR_OPCODE(BitwiseNorU32, General) -IR_OPCODE(BitwiseNorU64, General) -IR_OPCODE(BitwiseXnorU32, General) -IR_OPCODE(BitwiseXnorU64, General) -IR_OPCODE(BitwiseNotU32, General) -IR_OPCODE(BitwiseNotU64, General) -IR_OPCODE(BitClearU32, General) -IR_OPCODE(BitSetU32, General) -IR_OPCODE(BitReverseU32, General) -IR_OPCODE(BitCountU32, General) -IR_OPCODE(BitCountU64, General) -IR_OPCODE(BitReplicateB64B32, General) -IR_OPCODE(BitCountAddU32, General) -IR_OPCODE(MaskedBitCountLowU32, General) -IR_OPCODE(MaskedBitCountHighU32, General) -IR_OPCODE(FindLsbU32, General) -IR_OPCODE(FindMsbFromHighU32, General) -IR_OPCODE(FindMsbFromHighU64, General) -IR_OPCODE(BitFieldMaskU32, General) -IR_OPCODE(BitFieldMaskU64, General) -IR_OPCODE(BitFieldExtractU32, General) -IR_OPCODE(BitFieldExtractU64, General) -IR_OPCODE(BitFieldExtract3U32, General) -IR_OPCODE(BitFieldExtract3I32, General) -IR_OPCODE(BitFieldInsertSelectU32, General) -IR_OPCODE(BitCompare0B32, General) -IR_OPCODE(BitCompare1B32, General) -IR_OPCODE(AlignBitU32, General) -IR_OPCODE(ShiftLeftAddU32, General) -IR_OPCODE(AddShiftLeftU32, General) -IR_OPCODE(XorAddU32, General) -IR_OPCODE(ShiftLeftOrU32, General) -IR_OPCODE(ShiftLeftLogicalU32, General) -IR_OPCODE(ShiftLeftLogicalU64, General) -IR_OPCODE(ShiftLeftLogicalU16, General) -IR_OPCODE(ShiftRightLogicalU32, General) -IR_OPCODE(ShiftRightLogicalU64, General) -IR_OPCODE(ShiftRightLogicalU16, General) -IR_OPCODE(ShiftRightArithmeticI32, General) -IR_OPCODE(ShiftRightArithmeticI16, General) -IR_OPCODE(SelectU32, General) -IR_OPCODE(SelectMaskU32, General) -IR_OPCODE(SelectF32Bits, General) -IR_OPCODE(SelectMaskF32Bits, General) -IR_OPCODE(SelectU64, General) -IR_OPCODE(PackLowLowU16, General) -IR_OPCODE(PackLowHighU16, General) -IR_OPCODE(PackHighHighU16, General) -IR_OPCODE(CompareFalse, Compare) -IR_OPCODE(CompareTrue, Compare) -IR_OPCODE(CompareEqU32, Compare) -IR_OPCODE(CompareNeU32, Compare) -IR_OPCODE(CompareGtU32, Compare) -IR_OPCODE(CompareGeU32, Compare) -IR_OPCODE(CompareLtU32, Compare) -IR_OPCODE(CompareLeU32, Compare) -IR_OPCODE(CompareEqU64, Compare64) -IR_OPCODE(CompareNeU64, Compare64) -IR_OPCODE(CompareMaskEqU32, Compare) -IR_OPCODE(CompareMaskNeU32, Compare) -IR_OPCODE(CompareMaskGtU32, Compare) -IR_OPCODE(CompareMaskGeU32, Compare) -IR_OPCODE(CompareMaskLtU32, Compare) -IR_OPCODE(CompareMaskLeU32, Compare) -IR_OPCODE(CompareEqI32, Compare) -IR_OPCODE(CompareNeI32, Compare) -IR_OPCODE(CompareGtI32, Compare) -IR_OPCODE(CompareGeI32, Compare) -IR_OPCODE(CompareLtI32, Compare) -IR_OPCODE(CompareLeI32, Compare) -IR_OPCODE(CompareEqI16, Compare) -IR_OPCODE(CompareNeI16, Compare) -IR_OPCODE(CompareGtI16, Compare) -IR_OPCODE(CompareGeI16, Compare) -IR_OPCODE(CompareLtI16, Compare) -IR_OPCODE(CompareLeI16, Compare) -IR_OPCODE(CompareMaskEqI32, Compare) -IR_OPCODE(CompareMaskNeI32, Compare) -IR_OPCODE(CompareMaskGtI32, Compare) -IR_OPCODE(CompareMaskGeI32, Compare) -IR_OPCODE(CompareMaskLtI32, Compare) -IR_OPCODE(CompareMaskLeI32, Compare) -IR_OPCODE(CompareEqU16, Compare) -IR_OPCODE(CompareNeU16, Compare) -IR_OPCODE(CompareGtU16, Compare) -IR_OPCODE(CompareGeU16, Compare) -IR_OPCODE(CompareLtU16, Compare) -IR_OPCODE(CompareLeU16, Compare) -IR_OPCODE(CompareEqF32, Compare) -IR_OPCODE(CompareNeF32, Compare) -IR_OPCODE(CompareGtF32, Compare) -IR_OPCODE(CompareGeF32, Compare) -IR_OPCODE(CompareLtF32, Compare) -IR_OPCODE(CompareLeF32, Compare) -IR_OPCODE(CompareOrderedF32, Compare) -IR_OPCODE(CompareUnorderedF32, Compare) -IR_OPCODE(CompareUnordEqF32, Compare) -IR_OPCODE(CompareUnordNeF32, Compare) -IR_OPCODE(CompareUnordGtF32, Compare) -IR_OPCODE(CompareUnordGeF32, Compare) -IR_OPCODE(CompareUnordLtF32, Compare) -IR_OPCODE(CompareUnordLeF32, Compare) -IR_OPCODE(CompareClassF32, Compare) -IR_OPCODE(CompareEqF16, Compare) -IR_OPCODE(CompareNeF16, Compare) -IR_OPCODE(CompareGtF16, Compare) -IR_OPCODE(CompareGeF16, Compare) -IR_OPCODE(CompareLtF16, Compare) -IR_OPCODE(CompareLeF16, Compare) -IR_OPCODE(CompareUnordNeF16, Compare) -IR_OPCODE(CompareMaskEqF16, Compare) -IR_OPCODE(CompareMaskNeF16, Compare) -IR_OPCODE(CompareMaskGtF16, Compare) -IR_OPCODE(CompareMaskGeF16, Compare) -IR_OPCODE(CompareMaskLtF16, Compare) -IR_OPCODE(CompareMaskLeF16, Compare) -IR_OPCODE(CompareMaskUnordNeF16, Compare) -IR_OPCODE(CompareMaskUnordGeF16, Compare) -IR_OPCODE(CompareMaskEqF32, Compare) -IR_OPCODE(CompareMaskNeF32, Compare) -IR_OPCODE(CompareMaskGtF32, Compare) -IR_OPCODE(CompareMaskGeF32, Compare) -IR_OPCODE(CompareMaskLtF32, Compare) -IR_OPCODE(CompareMaskLeF32, Compare) -IR_OPCODE(CompareMaskUnordEqF32, Compare) -IR_OPCODE(CompareMaskUnordNeF32, Compare) -IR_OPCODE(CompareMaskUnordGtF32, Compare) -IR_OPCODE(CompareMaskUnordGeF32, Compare) -IR_OPCODE(CompareMaskUnordLtF32, Compare) -IR_OPCODE(CompareMaskUnordLeF32, Compare) -IR_OPCODE(ConvertByteU32ToF32, General) -IR_OPCODE(ConvertU32ToF32, General) -IR_OPCODE(ConvertI32ToF32, General) -IR_OPCODE(ConvertF32ToU32, General) -IR_OPCODE(ConvertF32ToI32, General) -IR_OPCODE(ConvertF32ToF16, General) -IR_OPCODE(ConvertF16ToF32, General) -IR_OPCODE(ConvertU16ToF16, General) -IR_OPCODE(ConvertF16ToU16, General) -IR_OPCODE(ConvertI16ToF16, General) -IR_OPCODE(ConvertF16ToI16, General) -IR_OPCODE(ConvertRoundPlusInfF32ToI32, General) -IR_OPCODE(ConvertFloorF32ToI32, General) -IR_OPCODE(ConvertI4ToOffsetF32, General) -IR_OPCODE(LdexpF32, General) -IR_OPCODE(PackF32ToF16Rtz, General) -IR_OPCODE(PackSnorm2x16F32, General) -IR_OPCODE(PackUnorm2x16F32, General) -IR_OPCODE(PackU16U32, General) -IR_OPCODE(PackU8F32, General) -IR_OPCODE(PackB32F16, General) -IR_OPCODE(PackedMadI16, General) -IR_OPCODE(PackedMulLoU16, General) -IR_OPCODE(PackedAddI16, General) -IR_OPCODE(PackedSubI16, General) -IR_OPCODE(PackedLshlrevB16, General) -IR_OPCODE(PackedLshrrevB16, General) -IR_OPCODE(PackedAshrrevI16, General) -IR_OPCODE(PackedMaxI16, General) -IR_OPCODE(PackedMinI16, General) -IR_OPCODE(PackedMadU16, General) -IR_OPCODE(PackedAddU16, General) -IR_OPCODE(PackedSubU16, General) -IR_OPCODE(PackedMaxU16, General) -IR_OPCODE(PackedMinU16, General) -IR_OPCODE(PackedAddF16, General) -IR_OPCODE(PackedMulF16, General) -IR_OPCODE(PackedMinF16, General) -IR_OPCODE(PackedMaxF16, General) -IR_OPCODE(PackedFmaF16, General) -IR_OPCODE(AddF16, General) -IR_OPCODE(SubF16, General) -IR_OPCODE(MulF16, General) -IR_OPCODE(MinF16, General) -IR_OPCODE(MaxF16, General) -IR_OPCODE(FmaF16, General) -IR_OPCODE(MadMixF16, General) -IR_OPCODE(IAddU16, General) -IR_OPCODE(ISubI16, General) -IR_OPCODE(IMinI16, General) -IR_OPCODE(IMaxI16, General) -IR_OPCODE(UMinU16, General) -IR_OPCODE(UMaxU16, General) -IR_OPCODE(RcpF32, General) -IR_OPCODE(FractF32, General) -IR_OPCODE(TruncF32, General) -IR_OPCODE(CeilF32, General) -IR_OPCODE(RoundEvenF32, General) -IR_OPCODE(FloorF32, General) -IR_OPCODE(Exp2F32, General) -IR_OPCODE(Log2F32, General) -IR_OPCODE(InverseSqrtF32, General) -IR_OPCODE(SqrtF32, General) -IR_OPCODE(RcpF16, General) -IR_OPCODE(SqrtF16, General) -IR_OPCODE(InverseSqrtF16, General) -IR_OPCODE(Log2F16, General) -IR_OPCODE(Exp2F16, General) -IR_OPCODE(FloorF16, General) -IR_OPCODE(CeilF16, General) -IR_OPCODE(TruncF16, General) -IR_OPCODE(RoundEvenF16, General) -IR_OPCODE(SinF32, General) -IR_OPCODE(CosF32, General) -IR_OPCODE(CubeIdF32, General) -IR_OPCODE(CubeScF32, General) -IR_OPCODE(CubeTcF32, General) -IR_OPCODE(CubeMaF32, General) -IR_OPCODE(FAddF32, General) -IR_OPCODE(FSubF32, General) -IR_OPCODE(FMulF32, General) -IR_OPCODE(FMinF32, General) -IR_OPCODE(FMaxF32, General) -IR_OPCODE(FMadF32, General) -IR_OPCODE(Dot2AccF32F16, General) -IR_OPCODE(FMin3F32, General) -IR_OPCODE(FMax3F32, General) -IR_OPCODE(FMed3F32, General) -IR_OPCODE(Min3F16, General) -IR_OPCODE(Max3F16, General) -IR_OPCODE(Med3F16, General) -IR_OPCODE(LoadSrtDword, General) -IR_OPCODE(SLoadDword, General) -IR_OPCODE(SBufferLoadDword, General) -IR_OPCODE(BufferLoadUbyte, General) -IR_OPCODE(BufferLoadSbyte, General) -IR_OPCODE(BufferLoadUshort, General) -IR_OPCODE(BufferLoadSshort, General) -IR_OPCODE(BufferLoadDword, General) -IR_OPCODE(BufferStoreByte, General) -IR_OPCODE(BufferStoreShort, General) -IR_OPCODE(BufferStoreDword, General) -IR_OPCODE(AtomicSwapU32, General) -IR_OPCODE(AtomicAddU32, General) -IR_OPCODE(AtomicSubU32, General) -IR_OPCODE(AtomicSMinI32, General) -IR_OPCODE(AtomicUMinU32, General) -IR_OPCODE(AtomicSMaxI32, General) -IR_OPCODE(AtomicUMaxU32, General) -IR_OPCODE(AtomicAndU32, General) -IR_OPCODE(AtomicOrU32, General) -IR_OPCODE(AtomicXorU32, General) -IR_OPCODE(AtomicFMinF32, General) -IR_OPCODE(AtomicFMaxF32, General) -IR_OPCODE(FlatLoadUbyte, General) -IR_OPCODE(FlatLoadSbyte, General) -IR_OPCODE(FlatLoadUshort, General) -IR_OPCODE(FlatLoadSshort, General) -IR_OPCODE(FlatLoadDword, General) -IR_OPCODE(FlatStoreByte, General) -IR_OPCODE(FlatStoreShort, General) -IR_OPCODE(FlatStoreDword, General) -IR_OPCODE(DsReadUbyte, General) -IR_OPCODE(DsReadSbyte, General) -IR_OPCODE(DsReadUshort, General) -IR_OPCODE(DsReadSshort, General) -IR_OPCODE(DsReadB32, General) -IR_OPCODE(DsWriteByte, General) -IR_OPCODE(DsWriteShort, General) -IR_OPCODE(DsWriteB32, General) -IR_OPCODE(DsMinF32, General) -IR_OPCODE(DsMaxF32, General) -IR_OPCODE(DsSwizzleB32, General) -IR_OPCODE(DsConsume, General) -IR_OPCODE(DsAppend, General) -IR_OPCODE(DsWriteAddtidB32, General) -IR_OPCODE(DsReadAddtidB32, General) -IR_OPCODE(ImageGetResinfo, General) -IR_OPCODE(ImageGetLod, General) -IR_OPCODE(ImageLoad, General) -IR_OPCODE(ImageStore, General) -IR_OPCODE(ImageSample, General) -IR_OPCODE(ImageGather4, General) -IR_OPCODE(LoadInputF32, General) -IR_OPCODE(Export, General) diff --git a/src/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp deleted file mode 100644 index 667c61ec1..000000000 --- a/src/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp +++ /dev/null @@ -1,235 +0,0 @@ -#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h" - -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -bool HasInput(const ShaderInfo& info, StageInputKind kind, uint32_t location) { - return std::any_of(info.inputs.begin(), info.inputs.end(), [=](const auto& input) { - return input.kind == kind && input.location == location; - }); -} - -void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components, - std::string name) { - if (!HasInput(info, kind, location)) { - info.inputs.push_back({kind, location, components, std::move(name)}); - } -} - -bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) { - return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) { - return output.kind == kind && output.index == index; - }); -} - -void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t location, - std::string name) { - if (!HasOutput(info, kind, index)) { - info.outputs.push_back({kind, index, location, std::move(name)}); - } -} - -bool NeedsLocalInvocationIndex(const Program& program) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == Opcode::DsWriteAddtidB32 || inst.op == Opcode::DsReadAddtidB32) { - return true; - } - } - } - return false; -} - -bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) { - auto Fail = [&](const char* message) { - if (error != nullptr) { - *error = message; - } - return false; - }; - switch (program.stage) { - case ShaderType::Vertex: - if (options.vertex == nullptr) { - return Fail("vertex shader info requires vertex metadata"); - } - if (options.vertex->resources_num < 0 || - options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) { - return Fail("vertex resource count is out of range"); - } - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == Opcode::LoadInputF32 && - (inst.input_info.chan >= 4 || - inst.input_info.attr >= - static_cast(options.vertex->resources_num))) { - return Fail("vertex input reference is out of range"); - } - } - } - return true; - case ShaderType::Pixel: - if (options.pixel == nullptr) { - return Fail("pixel shader info requires pixel metadata"); - } - return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) || - Fail("pixel input count is out of range"); - case ShaderType::Compute: - if (options.compute == nullptr) { - return Fail("compute shader info requires compute metadata"); - } - return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) || - Fail("compute thread ID count is out of range"); - default: return Fail("unsupported shader stage for info collection"); - } -} - -void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex, - ShaderInfo& info) { - AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex"); - AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex"); - if (vertex == nullptr) { - return; - } - uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {}; - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == Opcode::LoadInputF32 && - inst.input_info.attr < static_cast(vertex->resources_num) && - inst.input_info.attr < ShaderVertexInputInfo::RES_MAX) { - used_components[inst.input_info.attr] = - std::max(used_components[inst.input_info.attr], inst.input_info.chan + 1u); - } - } - } - for (uint32_t attr = 0; attr < static_cast(vertex->resources_num) && - attr < ShaderVertexInputInfo::RES_MAX; - attr++) { - if (used_components[attr] != 0) { - AddInput(info, StageInputKind::Parameter, attr, used_components[attr], - fmt::format("in_attr_{}", attr)); - } - } -} - -void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) { - if (pixel == nullptr) { - return; - } - if (pixel->HasPositionInput()) { - AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord"); - } - if (pixel->ps_front_face) { - AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing"); - } - for (uint32_t input = 0; input < pixel->input_num; input++) { - AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input)); - } -} - -void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo* compute, - ShaderInfo& info) { - if (compute != nullptr) { - if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) { - AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID"); - } - if (compute->thread_ids_num > 0) { - AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID"); - } - if (compute->thread_ids_num > 0 || compute->tg_size_en) { - AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); - } - if (compute->dispatch_thread_dimensions) { - AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID"); - } - } - if (NeedsLocalInvocationIndex(program)) { - AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); - } -} - -void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo& info) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op != Opcode::Export) { - continue; - } - if (inst.export_info.kind == ExportTargetKind::MrtZ) { - if (pixel != nullptr && (inst.export_info.en & 0x1u) != 0 && - pixel->ps_depth_export_enable) { - AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth"); - } - if (pixel != nullptr && (inst.export_info.en & 0x4u) != 0 && - pixel->ps_sample_mask_export_enable) { - AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask"); - } - continue; - } - if (inst.export_info.en == 0) { - continue; - } - switch (inst.export_info.kind) { - case ExportTargetKind::Position: - AddOutput(info, StageOutputKind::Position, inst.export_info.index, 0, - "out_position"); - break; - case ExportTargetKind::Parameter: - AddOutput(info, StageOutputKind::Parameter, inst.export_info.index, - inst.export_info.index, - fmt::format("out_param_{}", inst.export_info.index)); - break; - case ExportTargetKind::Mrt: - AddOutput(info, StageOutputKind::Mrt, inst.export_info.index, - inst.export_info.index, - fmt::format("out_mrt_{}", inst.export_info.index)); - break; - default: break; - } - } - } -} - -} // namespace - -bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) { - if (!program.resource_tracking_complete || program.shader_info_complete) { - if (error != nullptr) { - *error = !program.resource_tracking_complete ? "shader resources were not tracked" - : "shader info already collected"; - } - return false; - } - if (!ValidateOptions(program, options, error)) { - return false; - } - - auto next = program.info; - next.inputs.clear(); - next.outputs.clear(); - next.has_bitwise_xor = - std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) { - return std::any_of(block.instructions.begin(), block.instructions.end(), - [](const auto& inst) { - switch (inst.op) { - case Opcode::BitwiseXorU32: - case Opcode::BitwiseXor3U32: - case Opcode::XorAddU32: return true; - default: return false; - } - }); - }); - switch (program.stage) { - case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break; - case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break; - case ShaderType::Compute: CollectComputeInputs(program, options.compute, next); break; - default: return false; - } - CollectOutputs(program, options.pixel, next); - program.info = std::move(next); - program.shader_info_complete = true; - return true; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/SrtPatcher.cpp b/src/graphics/shader/recompiler/ir/SrtPatcher.cpp deleted file mode 100644 index 0cdc2ba17..000000000 --- a/src/graphics/shader/recompiler/ir/SrtPatcher.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "graphics/shader/recompiler/ir/SrtPatcher.h" - -#include -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { - -bool PatchSrtReads(Program& program, std::string* error) { - if (!program.srt_plan_complete || program.srt_patching_complete || - program.resource_tracking_complete) { - if (error != nullptr) { - if (!program.srt_plan_complete) { - *error = "SRT plan is not ready"; - } else if (program.srt_patching_complete) { - *error = "SRT reads already patched"; - } else { - *error = "resources already tracked"; - } - } - return false; - } - - std::map offsets; - std::map values; - for (const auto& read: program.srt.reads) { - if (read.value >= program.provenance.values.size() || - read.flat_offset >= program.srt.reads.size() || - (program.provenance.values[read.value].op != ScalarValueOp::ReadConst && - program.provenance.values[read.value].op != ScalarValueOp::ReadConstBuffer)) { - if (error != nullptr) { - *error = fmt::format("invalid SRT read value={} flat_offset={}", read.value, - read.flat_offset); - } - return false; - } - const auto [it, inserted] = offsets.emplace(read.value, read.flat_offset); - if (!inserted && it->second != read.flat_offset) { - if (error != nullptr) { - *error = fmt::format("SRT read value {} has conflicting flat offsets", read.value); - } - return false; - } - const auto [value, unique] = values.emplace(read.flat_offset, read.value); - if (!unique && value->second != read.value) { - if (error != nullptr) { - *error = fmt::format("SRT flat offset {} has conflicting values", read.flat_offset); - } - return false; - } - } - if (offsets.size() != program.srt.reads.size() || values.size() != program.srt.reads.size()) { - if (error != nullptr) { - *error = "SRT reads do not form a dense value-to-offset bijection"; - } - return false; - } - - std::set dynamic_reads; - for (const auto value: program.srt.dynamic_reads) { - if (value >= program.provenance.values.size() || offsets.contains(value) || - (program.provenance.values[value].op != ScalarValueOp::ReadConst && - program.provenance.values[value].op != ScalarValueOp::ReadConstBuffer) || - !dynamic_reads.insert(value).second) { - if (error != nullptr) { - *error = fmt::format("invalid dynamic SRT read value={}", value); - } - return false; - } - } - std::set dynamic_sources; - for (const auto source: program.srt.dynamic_sources) { - if (source == ScalarProvenance::Undefined || - (source > ScalarProvenance::Unknown && - source - 2u >= program.provenance.descriptors.size()) || - !dynamic_sources.insert(source).second) { - if (error != nullptr) { - *error = fmt::format("invalid dynamic descriptor source={}", source); - } - return false; - } - } - - struct Patch { - std::reference_wrapper inst; - uint32_t flat_offset = 0; - }; - std::vector patches; - std::map matched; - for (const auto& [value, offset]: offsets) { - matched.emplace(value, false); - } - for (auto& block: program.blocks) { - for (auto& inst: block.instructions) { - if (inst.op != Opcode::SLoadDword && inst.op != Opcode::SBufferLoadDword) { - continue; - } - if (const auto found = offsets.find(inst.scalar_value); found != offsets.end()) { - const auto value_op = program.provenance.values[found->first].op; - if ((inst.op == Opcode::SLoadDword) != (value_op == ScalarValueOp::ReadConst)) { - if (error != nullptr) { - *error = fmt::format("SRT read value {} has the wrong scalar-load producer", - found->first); - } - return false; - } - patches.push_back({std::ref(inst), found->second}); - matched[found->first] = true; - } - } - } - for (const auto& [value, found]: matched) { - if (!found) { - if (error != nullptr) { - *error = fmt::format("SRT read value {} has no scalar-load producer", value); - } - return false; - } - } - - for (const auto& patch: patches) { - auto& inst = patch.inst.get(); - inst.op = Opcode::LoadSrtDword; - inst.src_count = 1; - for (auto& src: inst.src) { - src = {}; - } - inst.src[0].kind = OperandKind::ImmediateU32; - inst.src[0].imm = patch.flat_offset; - inst.memory = {}; - } - program.srt_patching_complete = true; - return true; -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/SrtPatcher.h b/src/graphics/shader/recompiler/ir/SrtPatcher.h deleted file mode 100644 index b60376f96..000000000 --- a/src/graphics/shader/recompiler/ir/SrtPatcher.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ -#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ - -#include "graphics/shader/recompiler/ir/ShaderIR.h" - -namespace Libs::Graphics::ShaderRecompiler::IR { - -// Replaces each immediate ReadConst producer selected by the SRT plan with a dense flat-buffer -// dword load. Dynamic-offset reads remain explicit. On failure no instruction is changed. -bool PatchSrtReads(Program& program, std::string* error); - -} // namespace Libs::Graphics::ShaderRecompiler::IR - -#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ */ diff --git a/src/graphics/shader/recompiler/ir/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/SrtWalker.cpp deleted file mode 100644 index 548a489d0..000000000 --- a/src/graphics/shader/recompiler/ir/SrtWalker.cpp +++ /dev/null @@ -1,624 +0,0 @@ -#include "graphics/shader/recompiler/ir/SrtWalker.h" - -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" - -#include -#include -#include -#include - -namespace Libs::Graphics::ShaderRecompiler::IR { -namespace { - -constexpr uint64_t AddressMask = 0x0000ffffffffffffull; - -const char* StageName(ShaderType stage) { - switch (stage) { - case ShaderType::Vertex: return "vertex"; - case ShaderType::Pixel: return "pixel"; - case ShaderType::Fetch: return "fetch"; - case ShaderType::Compute: return "compute"; - default: return "unknown"; - } -} - -std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) { - return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash, - StageName(program.stage), pc, message); -} - -bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { - if (base > AddressMask) { - return false; - } - if (offset < 0) { - const auto magnitude = uint64_t {0} - static_cast(offset); - if (magnitude > base) { - return false; - } - result = base - magnitude; - return true; - } - const auto magnitude = static_cast(offset); - if (magnitude > AddressMask - base) { - return false; - } - result = base + magnitude; - return true; -} - -bool ApplyOperation(ScalarValueOp op, const std::array& args, uint32_t& result) { - const auto shift = args[1] & 31u; - switch (op) { - case ScalarValueOp::Add: result = args[0] + args[1]; break; - case ScalarValueOp::AddCarry: result = args[0] + args[1] + (args[2] & 1u); break; - case ScalarValueOp::Carry: - result = static_cast( - (static_cast(args[0]) + args[1] + (args[2] & 1u)) >> 32u); - break; - case ScalarValueOp::Sub: result = args[0] - args[1]; break; - case ScalarValueOp::SubBorrow: result = args[0] - args[1] - (args[2] & 1u); break; - case ScalarValueOp::Borrow: - result = static_cast(args[1]) + (args[2] & 1u) > args[0] ? 1u : 0u; - break; - case ScalarValueOp::Mul: result = args[0] * args[1]; break; - case ScalarValueOp::And: result = args[0] & args[1]; break; - case ScalarValueOp::AndNot: result = args[0] & ~args[1]; break; - case ScalarValueOp::Or: result = args[0] | args[1]; break; - case ScalarValueOp::OrNot: result = args[0] | ~args[1]; break; - case ScalarValueOp::Xor: result = args[0] ^ args[1]; break; - case ScalarValueOp::Not: result = ~args[0]; break; - case ScalarValueOp::ShiftLeft: result = args[0] << shift; break; - case ScalarValueOp::ShiftRight: result = args[0] >> shift; break; - case ScalarValueOp::ShiftRightArithmetic: - result = static_cast(static_cast(args[0]) >> shift); - break; - case ScalarValueOp::BitFieldMaskU32: { - const auto count = args[0] & 31u; - const auto offset = args[1] & 31u; - result = ((uint32_t {1} << count) - 1u) << offset; - break; - } - case ScalarValueOp::BitFieldMaskU64Low: - case ScalarValueOp::BitFieldMaskU64High: { - const auto count = args[0] & 63u; - const auto value = ((uint64_t {1} << count) - 1u) << (args[1] & 63u); - result = op == ScalarValueOp::BitFieldMaskU64Low ? static_cast(value) - : static_cast(value >> 32u); - break; - } - case ScalarValueOp::Add3: result = args[0] + args[1] + args[2]; break; - case ScalarValueOp::ShiftLeftAdd: result = (args[0] << shift) + args[2]; break; - case ScalarValueOp::ShiftLeftAddCarry: - result = - static_cast(((static_cast(args[0]) << shift) + args[2]) >> 32u); - break; - case ScalarValueOp::AddShiftLeft: result = (args[0] + args[1]) << (args[2] & 31u); break; - case ScalarValueOp::XorAdd: result = (args[0] ^ args[1]) + args[2]; break; - case ScalarValueOp::ShiftLeftOr: result = (args[0] << shift) | args[2]; break; - default: return false; - } - return true; -} - -class ConstantFolder { -public: - explicit ConstantFolder(const ScalarProvenance& provenance) - : m_provenance(provenance), m_values(provenance.values.size()), - m_state(provenance.values.size()) {} - - bool Fold(uint32_t id, uint32_t& result) { - if (id >= m_provenance.values.size() || m_state[id] == 1 || m_state[id] == 3) { - return false; - } - if (m_state[id] == 2) { - result = m_values[id]; - return true; - } - m_state[id] = 1; - const auto& value = m_provenance.values[id]; - uint32_t out = 0; - switch (value.op) { - case ScalarValueOp::Constant: out = value.imm; break; - case ScalarValueOp::Phi: - if (value.phi_args.empty() || !Fold(value.phi_args[0], out)) { - return Dynamic(id); - } - for (size_t i = 1; i < value.phi_args.size(); i++) { - uint32_t other = 0; - if (!Fold(value.phi_args[i], other) || other != out) { - return Dynamic(id); - } - } - break; - default: { - const auto count = ScalarValueArgCount(value.op); - if (count == 0 || count > 3 || value.op == ScalarValueOp::ReadConst) { - return Dynamic(id); - } - std::array args {}; - for (uint32_t i = 0; i < count; i++) { - if (!Fold(value.args[i], args[i])) { - return Dynamic(id); - } - } - if (!ApplyOperation(value.op, args, out)) { - return Dynamic(id); - } - break; - } - } - m_state[id] = 2; - m_values[id] = out; - result = out; - return true; - } - -private: - bool Dynamic(uint32_t id) { - m_state[id] = 3; - return false; - } - - const ScalarProvenance& m_provenance; - std::vector m_values; - std::vector m_state; -}; - -class PlanBuilder { -public: - explicit PlanBuilder(Program& program): m_program(program), m_folder(program.provenance) { - m_state.resize(program.provenance.values.size()); - } - - bool Run(std::string* error) { - m_program.srt = {}; - for (const auto& block: m_program.blocks) { - for (const auto& inst: block.instructions) { - if (!CollectDescriptor(inst.memory.resource_source, inst.pc, error) || - !CollectDescriptor(inst.memory.sampler_source, inst.pc, error)) { - return false; - } - } - } - for (const auto& block: m_program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == Opcode::SLoadDword && inst.scalar_value < m_state.size() && - m_state[inst.scalar_value] == 0) { - const auto& value = m_program.provenance.values[inst.scalar_value]; - uint32_t ignored = 0; - if (value.op == ScalarValueOp::ReadConst && - m_folder.Fold(value.args[2], ignored) && - !CollectValue(inst.scalar_value, inst.pc, error)) { - return false; - } - } - } - } - return true; - } - -private: - bool Fail(uint32_t pc, std::string* error, const std::string& message) const { - if (error != nullptr) { - *error = Diagnostic(m_program, pc, message); - } - return false; - } - - bool CollectDescriptor(uint32_t source, uint32_t use_pc, std::string* error) { - const auto* descriptor = GetDescriptorSource(m_program, source); - if (descriptor == nullptr) { - if (source <= ScalarProvenance::Unknown) { - return source == ScalarProvenance::Undefined || AddDynamicSource(source); - } - return Fail(use_pc, error, fmt::format("invalid descriptor source {}", source)); - } - if (!DescriptorSourceResolved(m_program, source) || - DescriptorNeedsControlFlow(*descriptor)) { - MarkDescriptor(*descriptor); - return AddDynamicSource(source); - } - for (uint32_t i = 0; i < descriptor->dword_count; i++) { - if (!CollectValue(descriptor->dwords[i], use_pc, error)) { - return false; - } - } - return true; - } - - void MarkDescriptor(const DescriptorValue& descriptor) { - for (uint32_t i = 0; i < descriptor.dword_count; i++) { - MarkValue(descriptor.dwords[i]); - } - } - - void MarkValue(uint32_t id) { - if (id >= m_state.size() || m_state[id] != 0) { - return; - } - m_state[id] = 2; - const auto& value = m_program.provenance.values[id]; - if (value.op == ScalarValueOp::Phi) { - for (const auto arg: value.phi_args) { - MarkValue(arg); - } - return; - } - for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { - MarkValue(value.args[i]); - } - } - - bool DescriptorNeedsControlFlow(const DescriptorValue& descriptor) { - std::vector visited(m_program.provenance.values.size()); - for (uint32_t i = 0; i < descriptor.dword_count; i++) { - if (ValueNeedsControlFlow(descriptor.dwords[i], visited)) { - return true; - } - } - return false; - } - - bool ValueNeedsControlFlow(uint32_t id, std::vector& visited) { - if (id >= m_program.provenance.values.size() || visited[id] != 0) { - return false; - } - visited[id] = 1; - const auto& value = m_program.provenance.values[id]; - if (value.op == ScalarValueOp::Phi) { - uint32_t ignored = 0; - return !m_folder.Fold(id, ignored); - } - for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { - if (ValueNeedsControlFlow(value.args[i], visited)) { - return true; - } - } - return false; - } - - bool AddDynamicSource(uint32_t source) { - if (std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(), - source) == m_program.srt.dynamic_sources.end()) { - m_program.srt.dynamic_sources.push_back(source); - } - return true; - } - - bool CollectValue(uint32_t id, uint32_t use_pc, std::string* error) { - if (id >= m_program.provenance.values.size()) { - return Fail(use_pc, error, fmt::format("invalid scalar value {}", id)); - } - if (m_state[id] == 2) { - return true; - } - if (m_state[id] == 1) { - return Fail(use_pc, error, fmt::format("cyclic scalar value {}", id)); - } - m_state[id] = 1; - const auto& value = m_program.provenance.values[id]; - if (value.op == ScalarValueOp::Phi) { - for (const auto arg: value.phi_args) { - if (!CollectValue(arg, use_pc, error)) { - return false; - } - } - } else { - for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { - if (!CollectValue(value.args[i], use_pc, error)) { - return false; - } - } - } - if (value.op == ScalarValueOp::ReadConst || value.op == ScalarValueOp::ReadConstBuffer) { - const auto offset_arg = value.op == ScalarValueOp::ReadConst ? 2u : 4u; - uint32_t ignored = 0; - if (m_folder.Fold(value.args[offset_arg], ignored)) { - m_program.srt.reads.push_back( - {id, static_cast(m_program.srt.reads.size()), use_pc}); - } else { - m_program.srt.dynamic_reads.push_back(id); - } - } - m_state[id] = 2; - return true; - } - - Program& m_program; - ConstantFolder m_folder; - std::vector m_state; -}; - -class Evaluator { -public: - Evaluator(const Program& program, const SrtRuntime& runtime, uint32_t use_pc) - : m_program(program), m_runtime(runtime), m_use_pc(use_pc), - m_values(program.provenance.values.size()), m_state(program.provenance.values.size()) {} - - void SetUsePc(uint32_t use_pc) { m_use_pc = use_pc; } - - bool Evaluate(uint32_t id, uint32_t& result, std::string* error) { - if (id >= m_program.provenance.values.size()) { - return Fail(error, fmt::format("invalid scalar value {}", id)); - } - if (m_state[id] == 2) { - result = m_values[id]; - return true; - } - if (m_state[id] == 1) { - return Fail(error, fmt::format("cyclic scalar value {}", id)); - } - m_state[id] = 1; - const auto& value = m_program.provenance.values[id]; - uint32_t out = 0; - switch (value.op) { - case ScalarValueOp::UserData: - if (value.imm < m_program.user_data_base || - value.imm - m_program.user_data_base >= m_runtime.user_data.size()) { - return Fail(error, fmt::format("user SGPR {} is unavailable", value.imm)); - } - out = m_runtime.user_data[value.imm - m_program.user_data_base]; - break; - case ScalarValueOp::Constant: out = value.imm; break; - case ScalarValueOp::PcRelativeLow: - out = static_cast(m_runtime.shader_base + value.imm); - break; - case ScalarValueOp::PcRelativeHigh: - out = static_cast((m_runtime.shader_base + value.imm) >> 32u); - break; - case ScalarValueOp::Phi: { - const auto first = std::find_if(value.phi_args.begin(), value.phi_args.end(), - [id](uint32_t arg) { return arg != id; }); - if (first == value.phi_args.end()) { - return Fail(error, fmt::format("empty scalar phi {}", id)); - } - if (!Evaluate(*first, out, error)) { - return false; - } - for (const auto arg: value.phi_args) { - if (arg == id || arg == *first) { - continue; - } - uint32_t other = 0; - if (!Evaluate(arg, other, error)) { - return false; - } - if (other != out) { - return Fail(error, - fmt::format("scalar phi {} has runtime-dependent values", id)); - } - } - break; - } - case ScalarValueOp::ReadConst: - case ScalarValueOp::ReadConstBuffer: - if (!Read(value, out, error)) { - return false; - } - break; - case ScalarValueOp::Undefined: - case ScalarValueOp::Unknown: - return Fail(error, fmt::format("scalar value {} is unresolved", id)); - default: - if (!EvaluateOperation(value, out, error)) { - return false; - } - break; - } - m_state[id] = 2; - m_values[id] = out; - result = out; - return true; - } - -private: - bool Fail(std::string* error, const std::string& message) const { - if (error != nullptr) { - *error = Diagnostic(m_program, m_use_pc, message); - } - return false; - } - - bool EvaluateOperation(const ScalarValue& value, uint32_t& result, std::string* error) { - const auto count = ScalarValueArgCount(value.op); - if (count == 0 || count > 3) { - return Fail(error, fmt::format("unsupported scalar operation {}", - static_cast(value.op))); - } - std::array args {}; - for (uint32_t i = 0; i < count; i++) { - if (!Evaluate(value.args[i], args[i], error)) { - return false; - } - } - return ApplyOperation(value.op, args, result) || - Fail(error, fmt::format("unsupported scalar operation {}", - static_cast(value.op))); - } - - bool Read(const ScalarValue& value, uint32_t& result, std::string* error) { - const bool buffer = value.op == ScalarValueOp::ReadConstBuffer; - uint32_t lo = 0; - uint32_t hi = 0; - uint32_t offset = 0; - if (!Evaluate(value.args[0], lo, error) || !Evaluate(value.args[1], hi, error) || - !Evaluate(value.args[buffer ? 4u : 2u], offset, error)) { - return false; - } - const auto base = (static_cast(hi) << 32u | lo) & AddressMask; - const auto immediate = static_cast(static_cast(value.imm)); - uint64_t address = 0; - if (buffer) { - uint32_t num_records = 0; - uint32_t ignored = 0; - if (!Evaluate(value.args[2], num_records, error) || - !Evaluate(value.args[3], ignored, error)) { - return false; - } - if (immediate < 0) { - return Fail(error, - fmt::format("ReadConstBuffer pc=0x{:08x} has negative immediate {}", - value.pc, immediate)); - } - const auto byte_offset = static_cast(immediate) + offset; - const auto aligned_offset = byte_offset & ~uint64_t {3}; - const auto stride = (hi >> 16u) & 0x3fffu; - const auto size = stride == 0 ? static_cast(num_records) - : static_cast(stride) * num_records; - if (aligned_offset > size || size - aligned_offset < sizeof(uint32_t)) { - return Fail(error, - fmt::format("ReadConstBuffer pc=0x{:08x} offset={} exceeds size={}", - value.pc, aligned_offset, size)); - } - address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; - } else { - const auto base_aligned = base & ~uint64_t {3}; - const auto immediate_aligned = immediate & ~int64_t {3}; - const auto relative = immediate_aligned + static_cast(offset & ~uint32_t {3}); - if (!AddSignedAddress(base_aligned, relative, address)) { - return Fail(error, - fmt::format("ReadConst pc=0x{:08x} address base=0x{:016x} offset={} " - "is outside the 48-bit address space", - value.pc, base_aligned, relative)); - } - } - if (m_runtime.read_memory != nullptr && - !m_runtime.read_memory(m_runtime.userdata, address, &result)) { - return Fail( - error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address)); - } - if (m_runtime.read_memory == nullptr) { - std::memcpy(&result, reinterpret_cast(address), sizeof(result)); - } - return true; - } - - const Program& m_program; - const SrtRuntime& m_runtime; - uint32_t m_use_pc; - std::vector m_values; - std::vector m_state; -}; - -} // namespace - -bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t& result) { - ConstantFolder folder(provenance); - return folder.Fold(value, result); -} - -bool BuildSrtPlan(Program& program, std::string* error) { - if (program.resource_tracking_complete) { - if (error != nullptr) { - *error = "cannot rebuild SRT plan after resource tracking"; - } - return false; - } - if (program.srt_patching_complete) { - if (error != nullptr) { - *error = "cannot rebuild SRT plan after SRT patching"; - } - return false; - } - program.srt_plan_complete = false; - if (!PlanBuilder(program).Run(error)) { - return false; - } - program.srt_plan_complete = true; - return true; -} - -bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, - const SrtRuntime& runtime, DescriptorValue& result, - std::string* error) { - const DescriptorSourceRequest request {source, use_pc}; - std::vector results; - if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) { - return false; - } - result = results[0]; - return true; -} - -static bool EvaluateRuntimeSourcesImpl(const Program& program, - std::span requests, - const SrtRuntime& runtime, - std::vector& results, - std::vector& flat, bool evaluate_flat, - std::string* error) { - if (!program.srt_plan_complete) { - if (error != nullptr) { - *error = Diagnostic(program, 0, "SRT plan is not ready"); - } - return false; - } - std::vector evaluated; - evaluated.reserve(requests.size()); - std::vector flattened(evaluate_flat ? program.srt.reads.size() : 0u); - Evaluator evaluator(program, runtime, 0); - for (const auto& request: requests) { - const auto* descriptor = GetDescriptorSource(program, request.source); - const auto dynamic = - std::find(program.srt.dynamic_sources.begin(), program.srt.dynamic_sources.end(), - request.source) != program.srt.dynamic_sources.end(); - if (descriptor == nullptr || dynamic) { - if (error != nullptr) { - *error = Diagnostic(program, request.use_pc, - fmt::format("descriptor source {} is {}", request.source, - dynamic ? "GPU-dynamic" : "unresolved")); - } - return false; - } - evaluator.SetUsePc(request.use_pc); - auto value = *descriptor; - for (uint32_t i = 0; i < descriptor->dword_count; i++) { - if (!evaluator.Evaluate(descriptor->dwords[i], value.dwords[i], error)) { - return false; - } - } - evaluated.push_back(value); - } - if (evaluate_flat) { - for (const auto& read: program.srt.reads) { - evaluator.SetUsePc(read.use_pc); - if (read.flat_offset >= flattened.size()) { - if (error != nullptr) { - *error = Diagnostic(program, read.use_pc, "invalid flat SRT offset"); - } - return false; - } - if (!evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) { - return false; - } - } - } - results = std::move(evaluated); - if (evaluate_flat) { - flat = std::move(flattened); - } - return true; -} - -bool EvaluateDescriptorSources(const Program& program, - std::span requests, - const SrtRuntime& runtime, std::vector& results, - std::string* error) { - std::vector ignored; - return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error); -} - -bool EvaluateRuntimeSources(const Program& program, - std::span requests, - const SrtRuntime& runtime, std::vector& results, - std::vector& flat, std::string* error) { - return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error); -} - -bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector& flat, - std::string* error) { - std::vector ignored; - return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error); -} - -} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/Type.cpp b/src/graphics/shader/recompiler/ir/Type.cpp new file mode 100644 index 000000000..52f25126b --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Type.cpp @@ -0,0 +1,55 @@ +#include "graphics/shader/recompiler/ir/Type.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +bool AreTypesCompatible(Type lhs, Type rhs) { + return lhs == rhs || lhs == Type::Opaque || rhs == Type::Opaque || TypesOverlap(lhs, rhs); +} + +std::string TypeName(Type type) { + static constexpr std::array names = { + "Opaque", + "ScalarReg", + "VectorReg", + "U1", + "U8", + "U16", + "U32", + "U64", + "F16", + "F32", + "U32x2", + "U32x3", + "U32x4", + "F32x2", + "F32x3", + "F32x4", + "SrtResource", + "BufferResource", + "AddressResource", + "ImageResource", + "SamplerResource", + "LdsResource", + "GdsResource", + "ImageAddress", + }; + const auto bits = static_cast(type); + if (bits == 0) { + return "Void"; + } + std::string result; + for (uint32_t index = 0; index < names.size(); index++) { + if ((bits & (1u << index)) == 0) { + continue; + } + if (!result.empty()) { + result += '|'; + } + result += names[index]; + } + return result; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/Type.h b/src/graphics/shader/recompiler/ir/Type.h new file mode 100644 index 000000000..6195ca89b --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Type.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +enum class Type : uint32_t { + Void = 0, + Opaque = 1u << 0u, + ScalarReg = 1u << 1u, + VectorReg = 1u << 2u, + U1 = 1u << 3u, + U8 = 1u << 4u, + U16 = 1u << 5u, + U32 = 1u << 6u, + U64 = 1u << 7u, + F16 = 1u << 8u, + F32 = 1u << 9u, + U32x2 = 1u << 10u, + U32x3 = 1u << 11u, + U32x4 = 1u << 12u, + F32x2 = 1u << 13u, + F32x3 = 1u << 14u, + F32x4 = 1u << 15u, + SrtResource = 1u << 16u, + BufferResource = 1u << 17u, + AddressResource = 1u << 18u, + ImageResource = 1u << 19u, + SamplerResource = 1u << 20u, + LdsResource = 1u << 21u, + GdsResource = 1u << 22u, + ImageAddress = 1u << 23u, +}; + +constexpr Type operator|(Type lhs, Type rhs) { + return static_cast(static_cast(lhs) | static_cast(rhs)); +} + +constexpr Type operator&(Type lhs, Type rhs) { + return static_cast(static_cast(lhs) & static_cast(rhs)); +} + +constexpr bool TypesOverlap(Type lhs, Type rhs) { + return (lhs & rhs) != Type::Void; +} + +[[nodiscard]] bool AreTypesCompatible(Type lhs, Type rhs); +[[nodiscard]] std::string TypeName(Type type); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/Value.cpp b/src/graphics/shader/recompiler/ir/Value.cpp new file mode 100644 index 000000000..a5c9a9baa --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Value.cpp @@ -0,0 +1,277 @@ +#include "graphics/shader/recompiler/ir/Value.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +Value::Value(Inst* value): type(Type::Opaque), inst(value) {} +Value::Value(ScalarReg value): type(Type::ScalarReg), scalar_reg(value) {} +Value::Value(VectorReg value): type(Type::VectorReg), vector_reg(value) {} +Value::Value(bool value): type(Type::U1), imm_u1(value) {} +Value::Value(uint8_t value): type(Type::U8), imm_u8(value) {} +Value::Value(uint16_t value): type(Type::U16), imm_u16(value) {} +Value::Value(uint32_t value): type(Type::U32), imm_u32(value) {} +Value::Value(uint64_t value): type(Type::U64), imm_u64(value) {} + +Value::Value(Type value_type, uint64_t bits): type(value_type), imm_u64(bits) {} + +Value Value::F16(uint16_t bits) { + return Value(Type::F16, bits); +} + +Value Value::F32(float value) { + return Value(Type::F32, std::bit_cast(value)); +} + +bool Value::IsEmpty() const { + return type == Type::Void; +} + +bool Value::IsImmediate() const { + return type != Type::Opaque; +} + +bool Value::IsIdentity() const { + return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Identity; +} + +bool Value::IsPhi() const { + return type == Type::Opaque && inst->GetOpcode() == ValueOpcode::Phi; +} + +Type Value::GetType() const { + if (IsPhi()) { + return inst->Flags(); + } + if (IsIdentity()) { + return inst->Arg(0).GetType(); + } + return type == Type::Opaque ? inst->GetType() : type; +} + +Inst* Value::Instruction() const { + EXIT_IF(type != Type::Opaque); + return inst; +} + +Inst* Value::TryInstruction() const { + return type == Type::Opaque ? inst : nullptr; +} + +Inst* Value::ResolveInstruction() const { + EXIT_IF(type != Type::Opaque); + return IsIdentity() ? inst->Arg(0).ResolveInstruction() : inst; +} + +Value Value::Resolve() const { + return IsIdentity() ? inst->Arg(0).Resolve() : *this; +} + +ScalarReg Value::ScalarRegister() const { + EXIT_IF(type != Type::ScalarReg); + return scalar_reg; +} + +VectorReg Value::VectorRegister() const { + EXIT_IF(type != Type::VectorReg); + return vector_reg; +} + +bool Value::U1() const { + EXIT_IF(type != Type::U1); + return imm_u1; +} + +uint8_t Value::U8() const { + EXIT_IF(type != Type::U8); + return imm_u8; +} + +uint16_t Value::U16() const { + EXIT_IF(type != Type::U16); + return imm_u16; +} + +uint32_t Value::U32() const { + EXIT_IF(type != Type::U32); + return imm_u32; +} + +uint64_t Value::U64() const { + EXIT_IF(type != Type::U64); + return imm_u64; +} + +uint16_t Value::F16Bits() const { + EXIT_IF(type != Type::F16); + return imm_u16; +} + +float Value::F32Value() const { + EXIT_IF(type != Type::F32); + return std::bit_cast(imm_u32); +} + +bool Value::operator==(const Value& other) const { + if (type != other.type) { + return false; + } + switch (type) { + case Type::Void: return true; + case Type::Opaque: return inst == other.inst; + case Type::ScalarReg: return scalar_reg == other.scalar_reg; + case Type::VectorReg: return vector_reg == other.vector_reg; + case Type::U1: return imm_u1 == other.imm_u1; + case Type::U8: return imm_u8 == other.imm_u8; + case Type::U16: + case Type::F16: return imm_u16 == other.imm_u16; + case Type::U32: + case Type::F32: return imm_u32 == other.imm_u32; + case Type::U64: return imm_u64 == other.imm_u64; + default: return false; + } +} + +Inst::Inst(ValueOpcode value_opcode, uint64_t value_flags) + : opcode(value_opcode), flags(value_flags) { + const auto count = NumArgsOf(opcode); + if (count != std::numeric_limits::max()) { + args.resize(count); + } +} + +Inst::~Inst() { + ClearArgs(); +} + +ValueOpcode Inst::GetOpcode() const { + return opcode; +} + +Type Inst::GetType() const { + if (opcode == ValueOpcode::Phi) { + return static_cast(flags); + } + if (opcode == ValueOpcode::Identity && !args.empty()) { + return args.front().GetType(); + } + return TypeOf(opcode); +} + +bool Inst::MayHaveSideEffects() const { + return HasSideEffects(opcode); +} + +bool Inst::HasUses() const { + return !uses.empty(); +} + +size_t Inst::UseCount() const { + return uses.size(); +} + +size_t Inst::NumArgs() const { + return args.size(); +} + +Value Inst::Arg(size_t index) const { + EXIT_IF(index >= args.size()); + return args[index]; +} + +Block* Inst::PhiBlock(size_t index) const { + EXIT_IF(opcode != ValueOpcode::Phi || index >= phi_blocks.size()); + return phi_blocks[index]; +} + +Block* Inst::Parent() const { + return parent; +} + +const std::vector& Inst::Uses() const { + return uses; +} + +void Inst::SetParent(Block* block) { + parent = block; +} + +void Inst::SetArg(size_t index, Value value) { + if (index >= args.size()) { + EXIT_IF(NumArgsOf(opcode) != std::numeric_limits::max()); + args.resize(index + 1); + } + const auto old = args[index]; + if (auto* old_inst = old.TryInstruction(); old_inst != nullptr) { + RemoveUse(old_inst, index); + } + args[index] = value; + if (auto* new_inst = value.TryInstruction(); new_inst != nullptr) { + AddUse(new_inst, index); + } +} + +void Inst::AddPhiOperand(Block* predecessor, Value value) { + EXIT_IF(opcode != ValueOpcode::Phi); + const auto index = args.size(); + args.push_back(value); + phi_blocks.push_back(predecessor); + if (auto* value_inst = value.TryInstruction(); value_inst != nullptr) { + AddUse(value_inst, index); + } +} + +void Inst::ReplaceUsesWith(Value replacement, bool preserve) { + const auto old_uses = uses; + for (const auto& use: old_uses) { + use.user->SetArg(use.operand, replacement); + } + Invalidate(); + if (preserve) { + ReplaceOpcode(ValueOpcode::Identity); + args.resize(1); + SetArg(0, replacement); + } +} + +void Inst::ReplaceOpcode(ValueOpcode value_opcode) { + opcode = value_opcode; + const auto count = NumArgsOf(opcode); + if (count != std::numeric_limits::max()) { + EXIT_IF(!args.empty() && args.size() != count); + args.resize(count); + } +} + +void Inst::Invalidate() { + ClearArgs(); + opcode = ValueOpcode::Void; +} + +void Inst::AddUse(Inst* used, size_t operand) { + const auto found = std::ranges::find_if( + used->uses, [&](const Use& use) { return use.user == this && use.operand == operand; }); + EXIT_IF(found != used->uses.end()); + used->uses.push_back({this, operand}); +} + +void Inst::RemoveUse(Inst* used, size_t operand) { + const auto found = std::ranges::find_if( + used->uses, [&](const Use& use) { return use.user == this && use.operand == operand; }); + EXIT_IF(found == used->uses.end()); + used->uses.erase(found); +} + +void Inst::ClearArgs() { + for (size_t index = 0; index < args.size(); index++) { + if (auto* value_inst = args[index].TryInstruction(); value_inst != nullptr) { + RemoveUse(value_inst, index); + } + } + args.clear(); + phi_blocks.clear(); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/Value.h b/src/graphics/shader/recompiler/ir/Value.h new file mode 100644 index 000000000..063e28a4a --- /dev/null +++ b/src/graphics/shader/recompiler/ir/Value.h @@ -0,0 +1,159 @@ +#pragma once + +#include "common/assert.h" +#include "graphics/shader/recompiler/ir/Reg.h" +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +class Block; +class Inst; + +class Value { +public: + Value() = default; + explicit Value(Inst* value); + explicit Value(ScalarReg value); + explicit Value(VectorReg value); + explicit Value(bool value); + explicit Value(uint8_t value); + explicit Value(uint16_t value); + explicit Value(uint32_t value); + explicit Value(uint64_t value); + + static Value F16(uint16_t bits); + static Value F32(float value); + + [[nodiscard]] bool IsEmpty() const; + [[nodiscard]] bool IsImmediate() const; + [[nodiscard]] bool IsIdentity() const; + [[nodiscard]] bool IsPhi() const; + [[nodiscard]] Type GetType() const; + + [[nodiscard]] Inst* Instruction() const; + [[nodiscard]] Inst* TryInstruction() const; + [[nodiscard]] Inst* ResolveInstruction() const; + [[nodiscard]] Value Resolve() const; + [[nodiscard]] ScalarReg ScalarRegister() const; + [[nodiscard]] VectorReg VectorRegister() const; + [[nodiscard]] bool U1() const; + [[nodiscard]] uint8_t U8() const; + [[nodiscard]] uint16_t U16() const; + [[nodiscard]] uint32_t U32() const; + [[nodiscard]] uint64_t U64() const; + [[nodiscard]] uint16_t F16Bits() const; + [[nodiscard]] float F32Value() const; + + bool operator==(const Value& other) const; + +private: + Type type = Type::Void; + union { + Inst* inst; + ScalarReg scalar_reg; + VectorReg vector_reg; + bool imm_u1; + uint8_t imm_u8; + uint16_t imm_u16; + uint32_t imm_u32; + uint64_t imm_u64; + }; + + explicit Value(Type type, uint64_t bits); +}; +static_assert(std::is_trivially_copyable_v); + +template +class TypedValue: public Value { +public: + TypedValue() = default; + + template + requires(TypesOverlap(type_, other_type)) + TypedValue(const TypedValue& value): Value(value) {} + + explicit TypedValue(Value value): Value(value) { + EXIT_IF(!AreTypesCompatible(value.GetType(), type_)); + } +}; + +using U1 = TypedValue; +using U8 = TypedValue; +using U16 = TypedValue; +using U32 = TypedValue; +using U64 = TypedValue; +using F16 = TypedValue; +using F32 = TypedValue; +using U32F32 = TypedValue; + +struct Use { + Inst* user = nullptr; + size_t operand = 0; + + bool operator==(const Use&) const = default; +}; + +class Inst { +public: + explicit Inst(ValueOpcode opcode, uint64_t flags = 0); + ~Inst(); + + Inst(const Inst&) = delete; + Inst& operator=(const Inst&) = delete; + Inst(Inst&&) = delete; + Inst& operator=(Inst&&) = delete; + + [[nodiscard]] ValueOpcode GetOpcode() const; + [[nodiscard]] Type GetType() const; + [[nodiscard]] bool MayHaveSideEffects() const; + [[nodiscard]] bool HasUses() const; + [[nodiscard]] size_t UseCount() const; + [[nodiscard]] size_t NumArgs() const; + [[nodiscard]] Value Arg(size_t index) const; + [[nodiscard]] Block* PhiBlock(size_t index) const; + [[nodiscard]] Block* Parent() const; + [[nodiscard]] const std::vector& Uses() const; + + void SetParent(Block* block); + void SetArg(size_t index, Value value); + void AddPhiOperand(Block* predecessor, Value value); + void ReplaceUsesWith(Value replacement, bool preserve = true); + void ReplaceOpcode(ValueOpcode opcode); + void Invalidate(); + + template + requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v) + [[nodiscard]] T Flags() const { + T result {}; + std::memcpy(&result, &flags, sizeof(result)); + return result; + } + + template + requires(sizeof(T) <= sizeof(uint64_t) && std::is_trivially_copyable_v) + void SetFlags(T value) { + flags = 0; + std::memcpy(&flags, &value, sizeof(value)); + } + +private: + void AddUse(Inst* used, size_t operand); + void RemoveUse(Inst* used, size_t operand); + void ClearArgs(); + + ValueOpcode opcode; + uint64_t flags; + Block* parent = nullptr; + std::vector args; + std::vector phi_blocks; + std::vector uses; +}; + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.cpp b/src/graphics/shader/recompiler/ir/ValueProgram.cpp new file mode 100644 index 000000000..8887618b3 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/ValueProgram.cpp @@ -0,0 +1,241 @@ +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +bool Fail(std::string* error, const std::string& message) { + if (error != nullptr) { + *error = message; + } + return false; +} + +bool IsRegisterGet(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::GetThreadBitScalarRegister: + case ValueOpcode::GetScalarRegister: + case ValueOpcode::GetVectorRegister: + case ValueOpcode::GetGotoVariable: + case ValueOpcode::GetScc: + case ValueOpcode::GetExec: + case ValueOpcode::GetExecLo: + case ValueOpcode::GetExecHi: + case ValueOpcode::GetVcc: + case ValueOpcode::GetVccLo: + case ValueOpcode::GetVccHi: + case ValueOpcode::GetM0: return true; + default: return false; + } +} + +bool IsRuntimeRead(ValueOpcode opcode) { + return opcode == ValueOpcode::LoadAddressU32 || opcode == ValueOpcode::ReadConstBuffer; +} + +bool EquivalentValue(const ValueProgram& program, Value left, Value right, + std::vector>& visited) { + left = left.Resolve(); + right = right.Resolve(); + if (left == right) { + return true; + } + if (left.IsImmediate() || right.IsImmediate() || left.GetType() != right.GetType()) { + return false; + } + const auto* lhs = left.TryInstruction(); + const auto* rhs = right.TryInstruction(); + if (lhs == nullptr || rhs == nullptr || lhs->GetOpcode() != rhs->GetOpcode() || + lhs->NumArgs() != rhs->NumArgs()) { + return false; + } + if (std::ranges::find(visited, std::pair {lhs, rhs}) != visited.end()) { + return true; + } + visited.emplace_back(lhs, rhs); + if (IsRuntimeRead(lhs->GetOpcode())) { + const auto li = lhs->Flags().index; + const auto ri = rhs->Flags().index; + if (li >= program.memory_info.size() || ri >= program.memory_info.size() || + program.memory_info[li] != program.memory_info[ri]) { + return false; + } + } else if (lhs->Flags() != rhs->Flags()) { + return false; + } + for (size_t index = 0; index < lhs->NumArgs(); index++) { + if (lhs->GetOpcode() == ValueOpcode::Phi && lhs->PhiBlock(index) != rhs->PhiBlock(index)) { + return false; + } + if (!EquivalentValue(program, lhs->Arg(index), rhs->Arg(index), visited)) { + return false; + } + } + return true; +} + +} // namespace + +ValueProgram::~ValueProgram() { + // Values may cross block boundaries. Detach all arguments before any block starts destroying + // its instruction storage so reverse-use links always point to live definitions. + for (auto* block: blocks) { + for (auto& inst: *block) { + inst.Invalidate(); + } + } +} + +bool EquivalentValue(const ValueProgram& program, Value left, Value right) { + std::vector> visited; + return EquivalentValue(program, left, right, visited); +} + +Value ResolveInvariantPhi(const ValueProgram& program, Value value) { + value = value.Resolve(); + const auto* root = value.TryInstruction(); + if (root == nullptr || root->GetOpcode() != ValueOpcode::Phi) { + return value; + } + Value invariant; + std::vector pending {value}; + std::unordered_set visited; + while (!pending.empty()) { + const auto current = pending.back().Resolve(); + pending.pop_back(); + const auto* inst = current.TryInstruction(); + if (inst != nullptr && inst->GetOpcode() == ValueOpcode::Phi) { + if (!visited.insert(inst).second) { + continue; + } + for (size_t index = 0; index < inst->NumArgs(); index++) { + pending.push_back(inst->Arg(index)); + } + continue; + } + if (invariant.IsEmpty()) { + invariant = current; + } else if (!EquivalentValue(program, invariant, current)) { + return {}; + } + } + return invariant; +} + +bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error) { + if (program.blocks.size() != program.block_info.size() || + program.blocks.size() != program.block_storage.size()) { + return Fail(error, "value IR block storage is inconsistent"); + } + for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { + const auto* block = program.blocks[block_index]; + if (block != program.block_storage[block_index].get()) { + return Fail(error, "value IR block pointer is inconsistent"); + } + for (const auto& inst: *block) { + if (inst.Parent() != block) { + return Fail(error, "value IR instruction has the wrong parent block"); + } + if (require_ssa && IsRegisterGet(inst.GetOpcode())) { + return Fail(error, fmt::format("register getter {} survived SSA rewrite", + ValueOpcodeName(inst.GetOpcode()))); + } + if (inst.GetOpcode() != ValueOpcode::Phi && inst.GetOpcode() != ValueOpcode::Identity && + inst.GetType() == Type::Opaque) { + return Fail(error, fmt::format("untyped opcode {} survived translation", + ValueOpcodeName(inst.GetOpcode()))); + } + for (size_t arg_index = 0; arg_index < inst.NumArgs(); arg_index++) { + const auto arg = inst.Arg(arg_index); + if (arg.IsEmpty()) { + return Fail(error, fmt::format("{} has an empty argument", + ValueOpcodeName(inst.GetOpcode()))); + } + if (const auto* definition = arg.TryInstruction(); + definition != nullptr && definition->Parent() == nullptr) { + return Fail(error, "value IR argument has a detached definition"); + } + if (const auto* definition = arg.TryInstruction(); definition != nullptr) { + const auto& uses = definition->Uses(); + const auto use = std::ranges::find_if(uses, [&](const Use& candidate) { + return candidate.user == &inst && candidate.operand == arg_index; + }); + if (use == uses.end()) { + return Fail(error, + fmt::format("{} argument {} is absent from {} reverse uses", + ValueOpcodeName(inst.GetOpcode()), arg_index, + ValueOpcodeName(definition->GetOpcode()))); + } + } + } + } + } + return true; +} + +std::string ValueProgramToString(const ValueProgram& program) { + std::map ids; + size_t next_id = 1; + for (const auto* block: program.blocks) { + for (const auto& inst: *block) { + ids.emplace(&inst, next_id++); + } + } + const auto value_string = [&](Value value) { + if (value.IsEmpty()) { + return std::string(""); + } + if (const auto* inst = value.TryInstruction(); inst != nullptr) { + return fmt::format("%{}", ids.at(inst)); + } + switch (value.GetType()) { + case Type::ScalarReg: return fmt::format("s{}", RegIndex(value.ScalarRegister())); + case Type::VectorReg: return fmt::format("v{}", RegIndex(value.VectorRegister())); + case Type::U1: return std::string(value.U1() ? "true" : "false"); + case Type::U8: return fmt::format("{}u8", value.U8()); + case Type::U16: return fmt::format("{}u16", value.U16()); + case Type::U32: return fmt::format("0x{:08x}", value.U32()); + case Type::U64: return fmt::format("0x{:016x}", value.U64()); + case Type::F16: return fmt::format("f16(0x{:04x})", value.F16Bits()); + case Type::F32: return fmt::format("{}f", value.F32Value()); + default: return fmt::format("<{}>", TypeName(value.GetType())); + } + }; + + std::string text = + fmt::format("mode={}\n", program.dispatcher_fallback ? "dispatcher" : "structured"); + for (size_t block_index = 0; block_index < program.blocks.size(); block_index++) { + text += fmt::format("Block ${} pc=0x{:08x}..0x{:08x}\n", block_index, + program.block_info[block_index].start_pc, + program.block_info[block_index].end_pc); + for (const auto& inst: *program.blocks[block_index]) { + const auto type = inst.GetType(); + if (type != Type::Void) { + text += + fmt::format(" %{:<5} = {}", ids.at(&inst), ValueOpcodeName(inst.GetOpcode())); + } else { + text += fmt::format(" {}", ValueOpcodeName(inst.GetOpcode())); + } + for (size_t index = 0; index < inst.NumArgs(); index++) { + text += index == 0 ? " " : ", "; + if (inst.GetOpcode() == ValueOpcode::Phi) { + const auto predecessor = + std::ranges::find(program.blocks, inst.PhiBlock(index)); + text += fmt::format("[{}, ${}]", value_string(inst.Arg(index)), + std::distance(program.blocks.begin(), predecessor)); + } else { + text += value_string(inst.Arg(index)); + } + } + text += + fmt::format(" ({}; uses={})\n", TypeName(Value(const_cast(&inst)).GetType()), + inst.UseCount()); + } + } + return text; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ValueProgram.h b/src/graphics/shader/recompiler/ir/ValueProgram.h new file mode 100644 index 000000000..d5324dbae --- /dev/null +++ b/src/graphics/shader/recompiler/ir/ValueProgram.h @@ -0,0 +1,64 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Block.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +struct ValueBlockInfo { + uint32_t id = 0; + uint32_t start_pc = 0; + uint32_t end_pc = 0; + CFG::Terminator terminator; + Value condition; + Value indirect_target; +}; + +struct DescriptorSource { + std::array dwords {}; + uint32_t dword_count = 0; + + bool operator==(const DescriptorSource& other) const = default; +}; + +struct SrtRead { + Value value; + uint32_t flat_offset = 0; + uint32_t use_pc = 0; + + bool operator==(const SrtRead& other) const = default; +}; + +struct ValueProgram { + ValueProgram() = default; + ~ValueProgram(); + + ValueProgram(const ValueProgram&) = delete; + ValueProgram& operator=(const ValueProgram&) = delete; + ValueProgram(ValueProgram&&) = default; + ValueProgram& operator=(ValueProgram&&) = default; + + std::vector> block_storage; + BlockList blocks; + std::vector block_info; + // Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and + // storage-image swizzles. Typed memory instructions carry a dense index into these shader-local + // tables until those fields are consumed by emission. + std::vector memory_info; + std::vector export_info; + std::vector descriptor_sources; + std::vector srt_reads; + std::vector dynamic_reads; + bool dispatcher_fallback = false; +}; + +bool ValidateValueProgram(const ValueProgram& program, bool require_ssa, std::string* error); +bool EquivalentValue(const ValueProgram& program, Value left, Value right); +Value ResolveInvariantPhi(const ValueProgram& program, Value value); +std::string ValueProgramToString(const ValueProgram& program); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/Opcodes.h b/src/graphics/shader/recompiler/ir/opcodes/Opcodes.h new file mode 100644 index 000000000..be62c27c4 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/opcodes/Opcodes.h @@ -0,0 +1,71 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Type.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +enum class Opcode { +#define IR_OPCODE(name, category, lowering, ...) name, +#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc" +#undef IR_OPCODE + Count, +}; + +enum class OpcodeClass { General, Compare, Compare64 }; +enum class LoweringClass { + Control, + Move, + Lane, + State, + Memory, + Attribute, + IntegerCompare, + Integer16Compare, + FloatCompare, + Conversion, + Integer16, + PackedInteger16, + PackedFloat16, + Float16, + Float, + U64Mask, + SimpleInteger, + ComposedInteger, + ExtendedInteger, +}; + +struct OpcodeInfo { + std::string_view name; + OpcodeClass opcode_class = OpcodeClass::General; + LoweringClass lowering_class = LoweringClass::Control; +}; + +inline constexpr OpcodeInfo OPCODE_INFO[] = { +#define IR_OPCODE(name, category, lowering, ...) \ + {#name, OpcodeClass::category, LoweringClass::lowering}, +#include "graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc" +#undef IR_OPCODE +}; +static_assert(std::size(OPCODE_INFO) == static_cast(Opcode::Count)); + +constexpr const OpcodeInfo& GetOpcodeInfo(Opcode opcode) { + return OPCODE_INFO[static_cast(opcode)]; +} + +constexpr std::string_view OpcodeName(Opcode opcode) { + return GetOpcodeInfo(opcode).name; +} + +constexpr bool IsCompareOpcode(Opcode opcode) { + return GetOpcodeInfo(opcode).opcode_class != OpcodeClass::General; +} + +constexpr bool IsCompare64Opcode(Opcode opcode) { + return GetOpcodeInfo(opcode).opcode_class == OpcodeClass::Compare64; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ShaderIROpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp similarity index 99% rename from src/graphics/shader/recompiler/ir/ShaderIROpcodes.cpp rename to src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp index 485d714f8..1da0bf344 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIROpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.cpp @@ -103,6 +103,7 @@ constexpr LowerMap LOWER_OPS[] = { {Decoder::Opcode::VCvtFlrI32F32, Opcode::ConvertFloorF32ToI32}, {Decoder::Opcode::VCvtOffF32I4, Opcode::ConvertI4ToOffsetF32}, {Decoder::Opcode::VRcpF32, Opcode::RcpF32}, + {Decoder::Opcode::VRcpIflagF32, Opcode::RcpIflagF32}, {Decoder::Opcode::VFractF32, Opcode::FractF32}, {Decoder::Opcode::VTruncF32, Opcode::TruncF32}, {Decoder::Opcode::VCeilF32, Opcode::CeilF32}, diff --git a/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc new file mode 100644 index 000000000..e7f9dac73 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/opcodes/ShaderIROpcodes.inc @@ -0,0 +1,343 @@ +// Semantic operations produced by RDNA2 instruction lowering. The typed frontend consumes this +// baseline set before optimization and resource planning; it must not grow without decoder support. +IR_OPCODE(ControlNop, General, Control, Void) +IR_OPCODE(Waitcnt, General, Control, Void) +IR_OPCODE(Barrier, General, Control, Void) +IR_OPCODE(Sendmsg, General, Control, Void) +IR_OPCODE(TtraceData, General, Control, Void) +IR_OPCODE(InstPrefetch, General, Control, Void) +IR_OPCODE(MoveU32, General, Move) +IR_OPCODE(MoveF32Bits, General, Move) +IR_OPCODE(MoveRelDestU32, General, Lane) +IR_OPCODE(MoveRelSourceU32, General, Lane) +IR_OPCODE(MoveU64, General, Move) +IR_OPCODE(WqmB64, General, Move) +IR_OPCODE(SaveexecB32, General, State) +IR_OPCODE(SaveexecB64, General, State) +IR_OPCODE(ReadFirstLaneU32, General, Lane) +IR_OPCODE(ReadLaneU32, General, Lane) +IR_OPCODE(WriteLaneU32, General, Lane) +IR_OPCODE(Permlane16B32, General, Lane) +IR_OPCODE(Permlanex16B32, General, Lane) +IR_OPCODE(AbsI32, General, SimpleInteger) +IR_OPCODE(IAddU32, General, SimpleInteger) +IR_OPCODE(IAddCarryU32, General, State) +IR_OPCODE(ISubBorrowU32, General, State) +IR_OPCODE(ScalarAddCarryU32, General, State) +IR_OPCODE(ScalarSubBorrowU32, General, State) +IR_OPCODE(ScalarSubBorrowCarryU32, General, State) +IR_OPCODE(ScalarSignedAddOverflowI32, General, State) +IR_OPCODE(ScalarSignedSubOverflowI32, General, State) +IR_OPCODE(ScalarShiftLeftAddCarryU32, General, State) +IR_OPCODE(ISubU32, General, SimpleInteger) +IR_OPCODE(IMulU32, General, SimpleInteger) +IR_OPCODE(UMulHighU32, General, SimpleInteger) +IR_OPCODE(SMulHighI32, General, SimpleInteger) +IR_OPCODE(IMadI24U32, General, ExtendedInteger) +IR_OPCODE(UMadU24U32, General, ExtendedInteger) +IR_OPCODE(UMadU64U32, General, ExtendedInteger) +IR_OPCODE(SadU32, General, ExtendedInteger) +IR_OPCODE(IAdd3U32, General, ExtendedInteger) +IR_OPCODE(IMulI24U32, General, ExtendedInteger) +IR_OPCODE(UMulU24U32, General, ExtendedInteger) +IR_OPCODE(IMinI32, General, SimpleInteger) +IR_OPCODE(IMaxI32, General, SimpleInteger) +IR_OPCODE(IMin3I32, General, SimpleInteger) +IR_OPCODE(IMax3I32, General, SimpleInteger) +IR_OPCODE(IMed3I32, General, SimpleInteger) +IR_OPCODE(UMinU32, General, SimpleInteger) +IR_OPCODE(UMaxU32, General, SimpleInteger) +IR_OPCODE(UMin3U32, General, SimpleInteger) +IR_OPCODE(UMax3U32, General, SimpleInteger) +IR_OPCODE(UMed3U32, General, SimpleInteger) +IR_OPCODE(BitwiseAndU32, General, SimpleInteger) +IR_OPCODE(BitwiseAndU64, General, U64Mask) +IR_OPCODE(BitwiseAndNotU32, General, ComposedInteger) +IR_OPCODE(BitwiseAndNotU64, General, U64Mask) +IR_OPCODE(BitwiseOrU32, General, SimpleInteger) +IR_OPCODE(BitwiseOrU64, General, U64Mask) +IR_OPCODE(BitwiseOrNotU32, General, ComposedInteger) +IR_OPCODE(BitwiseOrNotU64, General, U64Mask) +IR_OPCODE(BitwiseAndOrU32, General, ComposedInteger) +IR_OPCODE(BitwiseOr3U32, General, ComposedInteger) +IR_OPCODE(BitwiseXorU32, General, SimpleInteger) +IR_OPCODE(BitwiseXorU64, General, U64Mask) +IR_OPCODE(BitwiseXor3U32, General, ComposedInteger) +IR_OPCODE(BitwiseNandU32, General, ComposedInteger) +IR_OPCODE(BitwiseNandU64, General, U64Mask) +IR_OPCODE(BitwiseNorU32, General, ComposedInteger) +IR_OPCODE(BitwiseNorU64, General, U64Mask) +IR_OPCODE(BitwiseXnorU32, General, ComposedInteger) +IR_OPCODE(BitwiseXnorU64, General, U64Mask) +IR_OPCODE(BitwiseNotU32, General, SimpleInteger) +IR_OPCODE(BitwiseNotU64, General, U64Mask) +IR_OPCODE(BitClearU32, General, ExtendedInteger) +IR_OPCODE(BitSetU32, General, ExtendedInteger) +IR_OPCODE(BitReverseU32, General, SimpleInteger) +IR_OPCODE(BitCountU32, General, SimpleInteger) +IR_OPCODE(BitCountU64, General, SimpleInteger) +IR_OPCODE(BitReplicateB64B32, General, ExtendedInteger) +IR_OPCODE(BitCountAddU32, General, ExtendedInteger) +IR_OPCODE(MaskedBitCountLowU32, General, ExtendedInteger) +IR_OPCODE(MaskedBitCountHighU32, General, ExtendedInteger) +IR_OPCODE(FindLsbU32, General, SimpleInteger) +IR_OPCODE(FindMsbFromHighU32, General, ComposedInteger) +IR_OPCODE(FindMsbFromHighU64, General, ComposedInteger) +IR_OPCODE(BitFieldMaskU32, General, ExtendedInteger) +IR_OPCODE(BitFieldMaskU64, General, ExtendedInteger) +IR_OPCODE(BitFieldExtractU32, General, ExtendedInteger) +IR_OPCODE(BitFieldExtractU64, General, ExtendedInteger) +IR_OPCODE(BitFieldExtract3U32, General, ExtendedInteger) +IR_OPCODE(BitFieldExtract3I32, General, ExtendedInteger) +IR_OPCODE(BitFieldInsertSelectU32, General, ExtendedInteger) +IR_OPCODE(BitCompare0B32, General, ExtendedInteger) +IR_OPCODE(BitCompare1B32, General, ExtendedInteger) +IR_OPCODE(AlignBitU32, General, ExtendedInteger) +IR_OPCODE(ShiftLeftAddU32, General, ExtendedInteger) +IR_OPCODE(AddShiftLeftU32, General, ExtendedInteger) +IR_OPCODE(XorAddU32, General, ExtendedInteger) +IR_OPCODE(ShiftLeftOrU32, General, ExtendedInteger) +IR_OPCODE(ShiftLeftLogicalU32, General, SimpleInteger) +IR_OPCODE(ShiftLeftLogicalU64, General, SimpleInteger) +IR_OPCODE(ShiftLeftLogicalU16, General, Integer16) +IR_OPCODE(ShiftRightLogicalU32, General, SimpleInteger) +IR_OPCODE(ShiftRightLogicalU64, General, SimpleInteger) +IR_OPCODE(ShiftRightLogicalU16, General, Integer16) +IR_OPCODE(ShiftRightArithmeticI32, General, SimpleInteger) +IR_OPCODE(ShiftRightArithmeticI16, General, Integer16) +IR_OPCODE(SelectU32, General, ExtendedInteger, U32, U1, U32, U32) +IR_OPCODE(SelectMaskU32, General, ExtendedInteger) +IR_OPCODE(SelectF32Bits, General, ExtendedInteger) +IR_OPCODE(SelectMaskF32Bits, General, ExtendedInteger) +IR_OPCODE(SelectU64, General, ExtendedInteger) +IR_OPCODE(PackLowLowU16, General, ExtendedInteger) +IR_OPCODE(PackLowHighU16, General, ExtendedInteger) +IR_OPCODE(PackHighHighU16, General, ExtendedInteger) +IR_OPCODE(CompareFalse, Compare, IntegerCompare) +IR_OPCODE(CompareTrue, Compare, IntegerCompare) +IR_OPCODE(CompareEqU32, Compare, IntegerCompare) +IR_OPCODE(CompareNeU32, Compare, IntegerCompare) +IR_OPCODE(CompareGtU32, Compare, IntegerCompare) +IR_OPCODE(CompareGeU32, Compare, IntegerCompare) +IR_OPCODE(CompareLtU32, Compare, IntegerCompare) +IR_OPCODE(CompareLeU32, Compare, IntegerCompare) +IR_OPCODE(CompareEqU64, Compare64, IntegerCompare) +IR_OPCODE(CompareNeU64, Compare64, IntegerCompare) +IR_OPCODE(CompareMaskEqU32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskNeU32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskGtU32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskGeU32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskLtU32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskLeU32, Compare, IntegerCompare) +IR_OPCODE(CompareEqI32, Compare, IntegerCompare) +IR_OPCODE(CompareNeI32, Compare, IntegerCompare) +IR_OPCODE(CompareGtI32, Compare, IntegerCompare) +IR_OPCODE(CompareGeI32, Compare, IntegerCompare) +IR_OPCODE(CompareLtI32, Compare, IntegerCompare) +IR_OPCODE(CompareLeI32, Compare, IntegerCompare) +IR_OPCODE(CompareEqI16, Compare, Integer16Compare) +IR_OPCODE(CompareNeI16, Compare, Integer16Compare) +IR_OPCODE(CompareGtI16, Compare, Integer16Compare) +IR_OPCODE(CompareGeI16, Compare, Integer16Compare) +IR_OPCODE(CompareLtI16, Compare, Integer16Compare) +IR_OPCODE(CompareLeI16, Compare, Integer16Compare) +IR_OPCODE(CompareMaskEqI32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskNeI32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskGtI32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskGeI32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskLtI32, Compare, IntegerCompare) +IR_OPCODE(CompareMaskLeI32, Compare, IntegerCompare) +IR_OPCODE(CompareEqU16, Compare, Integer16Compare) +IR_OPCODE(CompareNeU16, Compare, Integer16Compare) +IR_OPCODE(CompareGtU16, Compare, Integer16Compare) +IR_OPCODE(CompareGeU16, Compare, Integer16Compare) +IR_OPCODE(CompareLtU16, Compare, Integer16Compare) +IR_OPCODE(CompareLeU16, Compare, Integer16Compare) +IR_OPCODE(CompareEqF32, Compare, FloatCompare) +IR_OPCODE(CompareNeF32, Compare, FloatCompare) +IR_OPCODE(CompareGtF32, Compare, FloatCompare) +IR_OPCODE(CompareGeF32, Compare, FloatCompare) +IR_OPCODE(CompareLtF32, Compare, FloatCompare) +IR_OPCODE(CompareLeF32, Compare, FloatCompare) +IR_OPCODE(CompareOrderedF32, Compare, FloatCompare) +IR_OPCODE(CompareUnorderedF32, Compare, FloatCompare) +IR_OPCODE(CompareUnordEqF32, Compare, FloatCompare) +IR_OPCODE(CompareUnordNeF32, Compare, FloatCompare) +IR_OPCODE(CompareUnordGtF32, Compare, FloatCompare) +IR_OPCODE(CompareUnordGeF32, Compare, FloatCompare) +IR_OPCODE(CompareUnordLtF32, Compare, FloatCompare) +IR_OPCODE(CompareUnordLeF32, Compare, FloatCompare) +IR_OPCODE(CompareClassF32, Compare, FloatCompare) +IR_OPCODE(CompareEqF16, Compare, FloatCompare) +IR_OPCODE(CompareNeF16, Compare, FloatCompare) +IR_OPCODE(CompareGtF16, Compare, FloatCompare) +IR_OPCODE(CompareGeF16, Compare, FloatCompare) +IR_OPCODE(CompareLtF16, Compare, FloatCompare) +IR_OPCODE(CompareLeF16, Compare, FloatCompare) +IR_OPCODE(CompareUnordNeF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskEqF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskNeF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskGtF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskGeF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskLtF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskLeF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordNeF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordGeF16, Compare, FloatCompare) +IR_OPCODE(CompareMaskEqF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskNeF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskGtF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskGeF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskLtF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskLeF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordEqF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordNeF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordGtF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordGeF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordLtF32, Compare, FloatCompare) +IR_OPCODE(CompareMaskUnordLeF32, Compare, FloatCompare) +IR_OPCODE(ConvertByteU32ToF32, General, Conversion) +IR_OPCODE(ConvertU32ToF32, General, Conversion) +IR_OPCODE(ConvertI32ToF32, General, Conversion) +IR_OPCODE(ConvertF32ToU32, General, Conversion) +IR_OPCODE(ConvertF32ToI32, General, Conversion) +IR_OPCODE(ConvertF32ToF16, General, Conversion) +IR_OPCODE(ConvertF16ToF32, General, Conversion) +IR_OPCODE(ConvertU16ToF16, General, Conversion) +IR_OPCODE(ConvertF16ToU16, General, Conversion) +IR_OPCODE(ConvertI16ToF16, General, Conversion) +IR_OPCODE(ConvertF16ToI16, General, Conversion) +IR_OPCODE(ConvertRoundPlusInfF32ToI32, General, Conversion) +IR_OPCODE(ConvertFloorF32ToI32, General, Conversion) +IR_OPCODE(ConvertI4ToOffsetF32, General, Conversion) +IR_OPCODE(LdexpF32, General, Float) +IR_OPCODE(PackF32ToF16Rtz, General, Conversion) +IR_OPCODE(PackSnorm2x16F32, General, Conversion) +IR_OPCODE(PackUnorm2x16F32, General, Conversion) +IR_OPCODE(PackU16U32, General, ExtendedInteger) +IR_OPCODE(PackU8F32, General, Conversion) +IR_OPCODE(PackB32F16, General, Conversion) +IR_OPCODE(PackedMadI16, General, PackedInteger16) +IR_OPCODE(PackedMulLoU16, General, PackedInteger16) +IR_OPCODE(PackedAddI16, General, PackedInteger16) +IR_OPCODE(PackedSubI16, General, PackedInteger16) +IR_OPCODE(PackedLshlrevB16, General, PackedInteger16) +IR_OPCODE(PackedLshrrevB16, General, PackedInteger16) +IR_OPCODE(PackedAshrrevI16, General, PackedInteger16) +IR_OPCODE(PackedMaxI16, General, PackedInteger16) +IR_OPCODE(PackedMinI16, General, PackedInteger16) +IR_OPCODE(PackedMadU16, General, PackedInteger16) +IR_OPCODE(PackedAddU16, General, PackedInteger16) +IR_OPCODE(PackedSubU16, General, PackedInteger16) +IR_OPCODE(PackedMaxU16, General, PackedInteger16) +IR_OPCODE(PackedMinU16, General, PackedInteger16) +IR_OPCODE(PackedAddF16, General, PackedFloat16) +IR_OPCODE(PackedMulF16, General, PackedFloat16) +IR_OPCODE(PackedMinF16, General, PackedFloat16) +IR_OPCODE(PackedMaxF16, General, PackedFloat16) +IR_OPCODE(PackedFmaF16, General, PackedFloat16) +IR_OPCODE(AddF16, General, Float16) +IR_OPCODE(SubF16, General, Float16) +IR_OPCODE(MulF16, General, Float16) +IR_OPCODE(MinF16, General, Float16) +IR_OPCODE(MaxF16, General, Float16) +IR_OPCODE(FmaF16, General, Float16) +IR_OPCODE(MadMixF16, General, Float16) +IR_OPCODE(IAddU16, General, Integer16) +IR_OPCODE(ISubI16, General, Integer16) +IR_OPCODE(IMinI16, General, Integer16) +IR_OPCODE(IMaxI16, General, Integer16) +IR_OPCODE(UMinU16, General, Integer16) +IR_OPCODE(UMaxU16, General, Integer16) +IR_OPCODE(RcpF32, General, Float) +IR_OPCODE(RcpIflagF32, General, Float) +IR_OPCODE(FractF32, General, Float) +IR_OPCODE(TruncF32, General, Float) +IR_OPCODE(CeilF32, General, Float) +IR_OPCODE(RoundEvenF32, General, Float) +IR_OPCODE(FloorF32, General, Float) +IR_OPCODE(Exp2F32, General, Float) +IR_OPCODE(Log2F32, General, Float) +IR_OPCODE(InverseSqrtF32, General, Float) +IR_OPCODE(SqrtF32, General, Float) +IR_OPCODE(RcpF16, General, Float16) +IR_OPCODE(SqrtF16, General, Float16) +IR_OPCODE(InverseSqrtF16, General, Float16) +IR_OPCODE(Log2F16, General, Float16) +IR_OPCODE(Exp2F16, General, Float16) +IR_OPCODE(FloorF16, General, Float16) +IR_OPCODE(CeilF16, General, Float16) +IR_OPCODE(TruncF16, General, Float16) +IR_OPCODE(RoundEvenF16, General, Float16) +IR_OPCODE(SinF32, General, Float) +IR_OPCODE(CosF32, General, Float) +IR_OPCODE(CubeIdF32, General, Float) +IR_OPCODE(CubeScF32, General, Float) +IR_OPCODE(CubeTcF32, General, Float) +IR_OPCODE(CubeMaF32, General, Float) +IR_OPCODE(FAddF32, General, Float) +IR_OPCODE(FSubF32, General, Float) +IR_OPCODE(FMulF32, General, Float) +IR_OPCODE(FMinF32, General, Float) +IR_OPCODE(FMaxF32, General, Float) +IR_OPCODE(FMadF32, General, Float) +IR_OPCODE(Dot2AccF32F16, General, Float) +IR_OPCODE(FMin3F32, General, Float) +IR_OPCODE(FMax3F32, General, Float) +IR_OPCODE(FMed3F32, General, Float) +IR_OPCODE(Min3F16, General, Float16) +IR_OPCODE(Max3F16, General, Float16) +IR_OPCODE(Med3F16, General, Float16) +IR_OPCODE(LoadSrtDword, General, Memory) +IR_OPCODE(SLoadDword, General, Memory) +IR_OPCODE(SBufferLoadDword, General, Memory) +IR_OPCODE(BufferLoadUbyte, General, Memory) +IR_OPCODE(BufferLoadSbyte, General, Memory) +IR_OPCODE(BufferLoadUshort, General, Memory) +IR_OPCODE(BufferLoadSshort, General, Memory) +IR_OPCODE(BufferLoadDword, General, Memory) +IR_OPCODE(BufferStoreByte, General, Memory) +IR_OPCODE(BufferStoreShort, General, Memory) +IR_OPCODE(BufferStoreDword, General, Memory) +IR_OPCODE(AtomicSwapU32, General, Memory) +IR_OPCODE(AtomicAddU32, General, Memory) +IR_OPCODE(AtomicSubU32, General, Memory) +IR_OPCODE(AtomicSMinI32, General, Memory) +IR_OPCODE(AtomicUMinU32, General, Memory) +IR_OPCODE(AtomicSMaxI32, General, Memory) +IR_OPCODE(AtomicUMaxU32, General, Memory) +IR_OPCODE(AtomicAndU32, General, Memory) +IR_OPCODE(AtomicOrU32, General, Memory) +IR_OPCODE(AtomicXorU32, General, Memory) +IR_OPCODE(AtomicFMinF32, General, Memory) +IR_OPCODE(AtomicFMaxF32, General, Memory) +IR_OPCODE(FlatLoadUbyte, General, Memory) +IR_OPCODE(FlatLoadSbyte, General, Memory) +IR_OPCODE(FlatLoadUshort, General, Memory) +IR_OPCODE(FlatLoadSshort, General, Memory) +IR_OPCODE(FlatLoadDword, General, Memory) +IR_OPCODE(FlatStoreByte, General, Memory) +IR_OPCODE(FlatStoreShort, General, Memory) +IR_OPCODE(FlatStoreDword, General, Memory) +IR_OPCODE(DsReadUbyte, General, Memory) +IR_OPCODE(DsReadSbyte, General, Memory) +IR_OPCODE(DsReadUshort, General, Memory) +IR_OPCODE(DsReadSshort, General, Memory) +IR_OPCODE(DsReadB32, General, Memory) +IR_OPCODE(DsWriteByte, General, Memory) +IR_OPCODE(DsWriteShort, General, Memory) +IR_OPCODE(DsWriteB32, General, Memory) +IR_OPCODE(DsMinF32, General, Memory) +IR_OPCODE(DsMaxF32, General, Memory) +IR_OPCODE(DsSwizzleB32, General, Memory) +IR_OPCODE(DsConsume, General, Memory) +IR_OPCODE(DsAppend, General, Memory) +IR_OPCODE(DsWriteAddtidB32, General, Memory) +IR_OPCODE(DsReadAddtidB32, General, Memory) +IR_OPCODE(ImageGetResinfo, General, Memory) +IR_OPCODE(ImageGetLod, General, Memory) +IR_OPCODE(ImageLoad, General, Memory) +IR_OPCODE(ImageStore, General, Memory) +IR_OPCODE(ImageSample, General, Memory) +IR_OPCODE(ImageGather4, General, Memory) +IR_OPCODE(LoadInputF32, General, Attribute) +IR_OPCODE(Export, General, Attribute) diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp new file mode 100644 index 000000000..c4c0d45f8 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -0,0 +1,144 @@ +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h" + +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +struct OpcodeMeta { + std::string_view name; + Type type = Type::Void; + std::array args = {}; + size_t num_args = 0; +}; + +template +consteval OpcodeMeta MakeMeta(std::string_view name, Type type, Args... args) { + static_assert(sizeof...(Args) <= 16); + OpcodeMeta meta {.name = name, .type = type, .num_args = sizeof...(Args)}; + meta.args.fill(Type::Void); + size_t index = 0; + ((meta.args[index++] = args), ...); + return meta; +} + +constexpr Type Void = Type::Void; +constexpr Type Opaque = Type::Opaque; +constexpr Type ScalarReg = Type::ScalarReg; +constexpr Type VectorReg = Type::VectorReg; +constexpr Type U1 = Type::U1; +constexpr Type U8 = Type::U8; +constexpr Type U16 = Type::U16; +constexpr Type U32 = Type::U32; +constexpr Type U64 = Type::U64; +constexpr Type F16 = Type::F16; +constexpr Type F32 = Type::F32; +constexpr Type U32x2 = Type::U32x2; +constexpr Type U32x4 = Type::U32x4; +constexpr Type F32x2 = Type::F32x2; +constexpr Type SrtResource = Type::SrtResource; +constexpr Type BufferResource = Type::BufferResource; +constexpr Type AddressResource = Type::AddressResource; +constexpr Type ImageResource = Type::ImageResource; +constexpr Type SamplerResource = Type::SamplerResource; +constexpr Type LdsResource = Type::LdsResource; +constexpr Type GdsResource = Type::GdsResource; +constexpr Type ImageAddress = Type::ImageAddress; + +constexpr std::array(ValueOpcode::Count)> MetaTable = {{ +#define VALUE_OPCODE(name, ...) MakeMeta(#name __VA_OPT__(, ) __VA_ARGS__), +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc" +#undef VALUE_OPCODE +}}; + +} // namespace + +Type TypeOf(ValueOpcode opcode) { + return MetaTable[static_cast(opcode)].type; +} + +size_t NumArgsOf(ValueOpcode opcode) { + return MetaTable[static_cast(opcode)].num_args; +} + +Type ArgTypeOf(ValueOpcode opcode, size_t index) { + const auto& meta = MetaTable[static_cast(opcode)]; + return meta.args.at(index); +} + +std::string_view ValueOpcodeName(ValueOpcode opcode) { + return MetaTable[static_cast(opcode)].name; +} + +bool HasSideEffects(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::Reference: + case ValueOpcode::ReferenceU32: + case ValueOpcode::Prologue: + case ValueOpcode::Epilogue: + case ValueOpcode::StoreAddressU8: + case ValueOpcode::StoreAddressU16: + case ValueOpcode::StoreAddressU32: + case ValueOpcode::StoreBufferU8: + case ValueOpcode::StoreBufferU16: + case ValueOpcode::StoreBufferU32: + case ValueOpcode::WriteSharedU8: + case ValueOpcode::WriteSharedU16: + case ValueOpcode::WriteSharedU32: + case ValueOpcode::WriteGdsU8: + case ValueOpcode::WriteGdsU16: + case ValueOpcode::WriteGdsU32: + case ValueOpcode::SharedAtomicFMin32: + case ValueOpcode::SharedAtomicFMax32: + case ValueOpcode::GdsAtomicFMin32: + case ValueOpcode::GdsAtomicFMax32: + case ValueOpcode::SharedAtomicSwap32: + case ValueOpcode::SharedAtomicIAdd32: + case ValueOpcode::SharedAtomicISub32: + case ValueOpcode::SharedAtomicSMin32: + case ValueOpcode::SharedAtomicUMin32: + case ValueOpcode::SharedAtomicSMax32: + case ValueOpcode::SharedAtomicUMax32: + case ValueOpcode::SharedAtomicAnd32: + case ValueOpcode::SharedAtomicOr32: + case ValueOpcode::SharedAtomicXor32: + case ValueOpcode::GdsAtomicSwap32: + case ValueOpcode::GdsAtomicIAdd32: + case ValueOpcode::GdsAtomicISub32: + case ValueOpcode::GdsAtomicSMin32: + case ValueOpcode::GdsAtomicUMin32: + case ValueOpcode::GdsAtomicSMax32: + case ValueOpcode::GdsAtomicUMax32: + case ValueOpcode::GdsAtomicAnd32: + case ValueOpcode::GdsAtomicOr32: + case ValueOpcode::GdsAtomicXor32: + case ValueOpcode::BufferAtomicSwap32: + case ValueOpcode::BufferAtomicIAdd32: + case ValueOpcode::BufferAtomicISub32: + case ValueOpcode::BufferAtomicSMin32: + case ValueOpcode::BufferAtomicUMin32: + case ValueOpcode::BufferAtomicSMax32: + case ValueOpcode::BufferAtomicUMax32: + case ValueOpcode::BufferAtomicAnd32: + case ValueOpcode::BufferAtomicOr32: + case ValueOpcode::BufferAtomicXor32: + case ValueOpcode::BufferAtomicFMin32: + case ValueOpcode::BufferAtomicFMax32: + case ValueOpcode::ImageAtomicIAdd32: + case ValueOpcode::ImageAtomicUMin32: + case ValueOpcode::ImageAtomicUMax32: + case ValueOpcode::ImageAtomicAnd32: + case ValueOpcode::ImageAtomicOr32: + case ValueOpcode::ImageAtomicXor32: + case ValueOpcode::DataAppend: + case ValueOpcode::DataConsume: + case ValueOpcode::GdsDataAppend: + case ValueOpcode::GdsDataConsume: + case ValueOpcode::ImageWrite: + case ValueOpcode::SetAttribute: + case ValueOpcode::Barrier: return true; + default: return false; + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h new file mode 100644 index 000000000..c86fb7cc0 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -0,0 +1,57 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Type.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +enum class ValueOpcode { +#define VALUE_OPCODE(name, ...) name, +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc" +#undef VALUE_OPCODE + Count, +}; + +struct DppMoveFlags { + uint16_t control = 0; + uint8_t row_mask = 0xf; + uint8_t bank_mask = 0xf; + bool fetch_inactive = false; + bool bound_control = false; +}; +static_assert(sizeof(DppMoveFlags) <= sizeof(uint64_t)); +static_assert(std::is_trivially_copyable_v); + +struct PermlaneFlags { + bool x16 = false; + bool fetch_inactive = false; + bool bound_control = false; +}; +static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t)); +static_assert(std::is_trivially_copyable_v); + +struct MemoryFlags { + uint32_t index = 0; + uint32_t pc = 0; +}; +static_assert(sizeof(MemoryFlags) == sizeof(uint64_t)); +static_assert(std::is_trivially_copyable_v); + +struct ExportFlags { + uint32_t index = 0; + uint32_t pc = 0; +}; +static_assert(sizeof(ExportFlags) == sizeof(uint64_t)); +static_assert(std::is_trivially_copyable_v); + +[[nodiscard]] Type TypeOf(ValueOpcode opcode); +[[nodiscard]] Type ArgTypeOf(ValueOpcode opcode, size_t index); +[[nodiscard]] size_t NumArgsOf(ValueOpcode opcode); +[[nodiscard]] bool HasSideEffects(ValueOpcode opcode); +[[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc new file mode 100644 index 000000000..327ff4396 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -0,0 +1,299 @@ +// operation, result, arguments... +VALUE_OPCODE(Phi, Opaque) +VALUE_OPCODE(Identity, Opaque, Opaque) +VALUE_OPCODE(Void, Void) +VALUE_OPCODE(ConditionRef, U1, U1) +VALUE_OPCODE(Reference, Void, U1) +VALUE_OPCODE(ReferenceU32, Void, U32) + +// Translation and register-state pseudo operations. +VALUE_OPCODE(Prologue, Void) +VALUE_OPCODE(Epilogue, Void) +VALUE_OPCODE(GetUserData, U32, ScalarReg) +VALUE_OPCODE(GetShaderBase, U64) +VALUE_OPCODE(GetBuiltin, U32, U32, U32) +VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg) +VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1) +VALUE_OPCODE(GetScalarRegister, U32, ScalarReg) +VALUE_OPCODE(SetScalarRegister, Void, ScalarReg, U32) +VALUE_OPCODE(GetVectorRegister, U32, VectorReg) +VALUE_OPCODE(SetVectorRegister, Void, VectorReg, U32) +VALUE_OPCODE(GetGotoVariable, U1, U32) +VALUE_OPCODE(SetGotoVariable, Void, U32, U1) +VALUE_OPCODE(GetScc, U1) +VALUE_OPCODE(SetScc, Void, U1) +VALUE_OPCODE(GetExec, U1) +VALUE_OPCODE(SetExec, Void, U1) +VALUE_OPCODE(GetExecLo, U32) +VALUE_OPCODE(SetExecLo, Void, U32) +VALUE_OPCODE(GetExecHi, U32) +VALUE_OPCODE(SetExecHi, Void, U32) +VALUE_OPCODE(GetVcc, U1) +VALUE_OPCODE(SetVcc, Void, U1) +VALUE_OPCODE(GetVccLo, U32) +VALUE_OPCODE(SetVccLo, Void, U32) +VALUE_OPCODE(GetVccHi, U32) +VALUE_OPCODE(SetVccHi, Void, U32) +VALUE_OPCODE(GetM0, U32) +VALUE_OPCODE(SetM0, Void, U32) + +VALUE_OPCODE(UndefU1, U1) +VALUE_OPCODE(UndefU8, U8) +VALUE_OPCODE(UndefU16, U16) +VALUE_OPCODE(UndefU32, U32) +VALUE_OPCODE(UndefU64, U64) + +// Value construction and bit-preserving conversion. +VALUE_OPCODE(BitCastU16F16, U16, F16) +VALUE_OPCODE(BitCastF16U16, F16, U16) +VALUE_OPCODE(BitCastU32F32, U32, F32) +VALUE_OPCODE(BitCastF32U32, F32, U32) +VALUE_OPCODE(ConvertU16U32, U16, U32) +VALUE_OPCODE(ConvertU32U16, U32, U16) +VALUE_OPCODE(ConvertU8U32, U8, U32) +VALUE_OPCODE(ConvertU32U8, U32, U8) +VALUE_OPCODE(ConvertF32F16, F32, F16) +VALUE_OPCODE(ConvertF16F32, F16, F32) +VALUE_OPCODE(ConvertS32F32, U32, F32) +VALUE_OPCODE(ConvertU32F32, U32, F32) +VALUE_OPCODE(ConvertF32S32, F32, U32) +VALUE_OPCODE(ConvertF32U32, F32, U32) +VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32) +VALUE_OPCODE(CompositeConstructF32x2, F32x2, F32, F32) +VALUE_OPCODE(CompositeConstructU32x4, U32x4, U32, U32, U32, U32) +VALUE_OPCODE(CompositeExtractU32x2, U32, U32x2, U32) +VALUE_OPCODE(CompositeExtractU32x4, U32, U32x4, U32) +VALUE_OPCODE(PackUint2x32, U64, U32x2) +VALUE_OPCODE(UnpackUint2x32, U32x2, U64) +VALUE_OPCODE(PackHalf2x16, U32, F32x2) +VALUE_OPCODE(PackSnorm2x16, U32, F32x2) +VALUE_OPCODE(PackUnorm2x16, U32, F32x2) +VALUE_OPCODE(PackFloat2x16Rtz, U32, F32, F32) + +// Source/destination modifiers made explicit by translation. +VALUE_OPCODE(FPAbs32, F32, F32) +VALUE_OPCODE(FPNeg32, F32, F32) +VALUE_OPCODE(FPSaturate32, F32, F32) +VALUE_OPCODE(BitFieldInsert, U32, U32, U32, U32, U32) +VALUE_OPCODE(BitFieldUExtract, U32, U32, U32, U32) +VALUE_OPCODE(BitFieldSExtract, U32, U32, U32, U32) +VALUE_OPCODE(DppMoveU32, U32, U32, U1) +VALUE_OPCODE(DppUpdateU32, U32, U32, U32, U1) +VALUE_OPCODE(WqmMask, U1, U1) + +// Core value operations used to expand architectural side effects. +VALUE_OPCODE(SelectU1, U1, U1, U1, U1) +VALUE_OPCODE(SelectF32, F32, U1, F32, F32) +VALUE_OPCODE(IAdd32, U32, U32, U32) +VALUE_OPCODE(IAdd64, U64, U64, U64) +VALUE_OPCODE(IAddCarry32, U32x2, U32, U32) +VALUE_OPCODE(ISub32, U32, U32, U32) +VALUE_OPCODE(ISub64, U64, U64, U64) +VALUE_OPCODE(IMul32, U32, U32, U32) +VALUE_OPCODE(IMul64, U64, U64, U64) +VALUE_OPCODE(UDiv32, U32, U32, U32) +VALUE_OPCODE(SMulHi, U32, U32, U32) +VALUE_OPCODE(UMulHi, U32, U32, U32) +VALUE_OPCODE(IAbs32, U32, U32) +VALUE_OPCODE(ShiftLeftLogical32, U32, U32, U32) +VALUE_OPCODE(ShiftLeftLogical64, U64, U64, U32) +VALUE_OPCODE(ShiftRightLogical32, U32, U32, U32) +VALUE_OPCODE(ShiftRightLogical64, U64, U64, U32) +VALUE_OPCODE(ShiftRightArithmetic32, U32, U32, U32) +VALUE_OPCODE(ShiftRightArithmetic64, U64, U64, U32) +VALUE_OPCODE(BitwiseAnd32, U32, U32, U32) +VALUE_OPCODE(BitwiseAnd64, U64, U64, U64) +VALUE_OPCODE(BitwiseOr32, U32, U32, U32) +VALUE_OPCODE(BitwiseOr64, U64, U64, U64) +VALUE_OPCODE(BitwiseXor32, U32, U32, U32) +VALUE_OPCODE(BitwiseNot32, U32, U32) +VALUE_OPCODE(BitReverse32, U32, U32) +VALUE_OPCODE(BitCount32, U32, U32) +VALUE_OPCODE(BitCount64, U32, U64) +VALUE_OPCODE(FindUMsb32, U32, U32) +VALUE_OPCODE(FindUMsb64, U32, U64) +VALUE_OPCODE(FindSMsb32, U32, U32) +VALUE_OPCODE(FindILsb32, U32, U32) +VALUE_OPCODE(SMin32, U32, U32, U32) +VALUE_OPCODE(UMin32, U32, U32, U32) +VALUE_OPCODE(SMax32, U32, U32, U32) +VALUE_OPCODE(UMax32, U32, U32, U32) +VALUE_OPCODE(SMinTri32, U32, U32, U32, U32) +VALUE_OPCODE(UMinTri32, U32, U32, U32, U32) +VALUE_OPCODE(SMaxTri32, U32, U32, U32, U32) +VALUE_OPCODE(UMaxTri32, U32, U32, U32, U32) +VALUE_OPCODE(SMedTri32, U32, U32, U32, U32) +VALUE_OPCODE(UMedTri32, U32, U32, U32, U32) +VALUE_OPCODE(SLessThan32, U1, U32, U32) +VALUE_OPCODE(SLessThan64, U1, U64, U64) +VALUE_OPCODE(ULessThan32, U1, U32, U32) +VALUE_OPCODE(ULessThan64, U1, U64, U64) +VALUE_OPCODE(IEqual32, U1, U32, U32) +VALUE_OPCODE(IEqual64, U1, U64, U64) +VALUE_OPCODE(SLessThanEqual32, U1, U32, U32) +VALUE_OPCODE(SLessThanEqual64, U1, U64, U64) +VALUE_OPCODE(ULessThanEqual32, U1, U32, U32) +VALUE_OPCODE(ULessThanEqual64, U1, U64, U64) +VALUE_OPCODE(SGreaterThan32, U1, U32, U32) +VALUE_OPCODE(SGreaterThan64, U1, U64, U64) +VALUE_OPCODE(UGreaterThan32, U1, U32, U32) +VALUE_OPCODE(UGreaterThan64, U1, U64, U64) +VALUE_OPCODE(INotEqual32, U1, U32, U32) +VALUE_OPCODE(INotEqual64, U1, U64, U64) +VALUE_OPCODE(SGreaterThanEqual32, U1, U32, U32) +VALUE_OPCODE(SGreaterThanEqual64, U1, U64, U64) +VALUE_OPCODE(UGreaterThanEqual32, U1, U32, U32) +VALUE_OPCODE(UGreaterThanEqual64, U1, U64, U64) +VALUE_OPCODE(LogicalOr, U1, U1, U1) +VALUE_OPCODE(LogicalAnd, U1, U1, U1) +VALUE_OPCODE(LogicalXor, U1, U1, U1) +VALUE_OPCODE(LogicalNot, U1, U1) +VALUE_OPCODE(FPOrdEqual32, U1, F32, F32) +VALUE_OPCODE(FPUnordEqual32, U1, F32, F32) +VALUE_OPCODE(FPOrdNotEqual32, U1, F32, F32) +VALUE_OPCODE(FPUnordNotEqual32, U1, F32, F32) +VALUE_OPCODE(FPOrdLessThan32, U1, F32, F32) +VALUE_OPCODE(FPUnordLessThan32, U1, F32, F32) +VALUE_OPCODE(FPOrdGreaterThan32, U1, F32, F32) +VALUE_OPCODE(FPUnordGreaterThan32, U1, F32, F32) +VALUE_OPCODE(FPOrdLessThanEqual32, U1, F32, F32) +VALUE_OPCODE(FPUnordLessThanEqual32, U1, F32, F32) +VALUE_OPCODE(FPOrdGreaterThanEqual32, U1, F32, F32) +VALUE_OPCODE(FPUnordGreaterThanEqual32, U1, F32, F32) +VALUE_OPCODE(FPIsNan32, U1, F32) +VALUE_OPCODE(FPCmpClass32, U1, F32, U32) + +VALUE_OPCODE(FPAdd32, F32, F32, F32) +VALUE_OPCODE(FPSub32, F32, F32, F32) +VALUE_OPCODE(FPFma32, F32, F32, F32, F32) +VALUE_OPCODE(FPMul32, F32, F32, F32) +VALUE_OPCODE(FPMin32, F32, F32, F32) +VALUE_OPCODE(FPMax32, F32, F32, F32) +VALUE_OPCODE(FPMinTri32, F32, F32, F32, F32) +VALUE_OPCODE(FPMaxTri32, F32, F32, F32, F32) +VALUE_OPCODE(FPMedTri32, F32, F32, F32, F32) +VALUE_OPCODE(FPRecip32, F32, F32) +VALUE_OPCODE(FPRecipIFlag32, F32, F32) +VALUE_OPCODE(FPRecipSqrt32, F32, F32) +VALUE_OPCODE(FPSqrt, F32, F32) +VALUE_OPCODE(FPSin, F32, F32) +VALUE_OPCODE(FPCos, F32, F32) +VALUE_OPCODE(FPExp2, F32, F32) +VALUE_OPCODE(FPLog2, F32, F32) +VALUE_OPCODE(FPLdexp, F32, F32, U32) +VALUE_OPCODE(FPRoundEven32, F32, F32) +VALUE_OPCODE(FPFloor32, F32, F32) +VALUE_OPCODE(FPCeil32, F32, F32) +VALUE_OPCODE(FPTrunc32, F32, F32) +VALUE_OPCODE(FPFract32, F32, F32) + +VALUE_OPCODE(LaneId, U32) +VALUE_OPCODE(Ballot, U32x4, U1) +VALUE_OPCODE(ReadFirstLane, U32, U32, U1) +VALUE_OPCODE(ReadLane, U32, U32, U32) +VALUE_OPCODE(WriteLane, U32, U32, U32, U32) +VALUE_OPCODE(Permlane16U32, U32, U32, U32, U32, U1) + +// Typed shader resources and memory operations. Descriptor state is represented by SSA values; +// dense bindings are assigned by resource tracking. +VALUE_OPCODE(GetSrtResource, SrtResource) +VALUE_OPCODE(GetBufferResource, BufferResource, U32, U32, U32, U32) +VALUE_OPCODE(GetAddressResource, AddressResource, U32, U32) +VALUE_OPCODE(GetImageResource, ImageResource, U32, U32, U32, U32, U32, U32, U32, U32) +VALUE_OPCODE(GetSamplerResource, SamplerResource, U32, U32, U32, U32) +VALUE_OPCODE(GetLdsResource, LdsResource) +VALUE_OPCODE(GetGdsResource, GdsResource) +VALUE_OPCODE(MakeImageAddress, ImageAddress, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, U32, + U32, U32) + +VALUE_OPCODE(ReadConst, U32, SrtResource, U32) +VALUE_OPCODE(ReadConstBuffer, U32, BufferResource, U32) +VALUE_OPCODE(LoadAddressU8, U8, AddressResource, U32, U32, U1) +VALUE_OPCODE(LoadAddressU16, U16, AddressResource, U32, U32, U1) +VALUE_OPCODE(LoadAddressU32, U32, AddressResource, U32, U32, U1) +VALUE_OPCODE(StoreAddressU8, Void, AddressResource, U32, U32, U8, U1) +VALUE_OPCODE(StoreAddressU16, Void, AddressResource, U32, U32, U16, U1) +VALUE_OPCODE(StoreAddressU32, Void, AddressResource, U32, U32, U32, U1) +VALUE_OPCODE(LoadBufferU8, U8, BufferResource, U32, U32, U32, U1) +VALUE_OPCODE(LoadBufferU16, U16, BufferResource, U32, U32, U32, U1) +VALUE_OPCODE(LoadBufferU32, U32, BufferResource, U32, U32, U32, U1) +VALUE_OPCODE(StoreBufferU8, Void, BufferResource, U32, U32, U32, U8, U1) +VALUE_OPCODE(StoreBufferU16, Void, BufferResource, U32, U32, U32, U16, U1) +VALUE_OPCODE(StoreBufferU32, Void, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicSwap32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicIAdd32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicISub32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicSMin32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicUMin32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicSMax32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicUMax32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicAnd32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicOr32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicXor32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicFMin32, U32, BufferResource, U32, U32, U32, U32, U1) +VALUE_OPCODE(BufferAtomicFMax32, U32, BufferResource, U32, U32, U32, U32, U1) + +VALUE_OPCODE(LoadSharedU8, U8, LdsResource, U32, U1) +VALUE_OPCODE(LoadSharedU16, U16, LdsResource, U32, U1) +VALUE_OPCODE(LoadSharedU32, U32, LdsResource, U32, U1) +VALUE_OPCODE(WriteSharedU8, Void, LdsResource, U32, U8, U1) +VALUE_OPCODE(WriteSharedU16, Void, LdsResource, U32, U16, U1) +VALUE_OPCODE(WriteSharedU32, Void, LdsResource, U32, U32, U1) +VALUE_OPCODE(LoadGdsU8, U8, GdsResource, U32, U1) +VALUE_OPCODE(LoadGdsU16, U16, GdsResource, U32, U1) +VALUE_OPCODE(LoadGdsU32, U32, GdsResource, U32, U1) +VALUE_OPCODE(WriteGdsU8, Void, GdsResource, U32, U8, U1) +VALUE_OPCODE(WriteGdsU16, Void, GdsResource, U32, U16, U1) +VALUE_OPCODE(WriteGdsU32, Void, GdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicFMin32, Void, LdsResource, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicFMax32, Void, LdsResource, U32, U32, U32, U1) +VALUE_OPCODE(GdsAtomicFMin32, Void, GdsResource, U32, U32, U32, U1) +VALUE_OPCODE(GdsAtomicFMax32, Void, GdsResource, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicSwap32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicIAdd32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicISub32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicSMin32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicUMin32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicSMax32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicUMax32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicAnd32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicOr32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(SharedAtomicXor32, U32, LdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicSwap32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicIAdd32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicISub32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicSMin32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicUMin32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicSMax32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicUMax32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicAnd32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicOr32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(GdsAtomicXor32, U32, GdsResource, U32, U32, U1) +VALUE_OPCODE(DataAppend, U32, LdsResource, U32, U1, U32, U32) +VALUE_OPCODE(DataConsume, U32, LdsResource, U32, U1, U32, U32) +VALUE_OPCODE(GdsDataAppend, U32, GdsResource, U32, U1, U32, U32) +VALUE_OPCODE(GdsDataConsume, U32, GdsResource, U32, U1, U32, U32) +VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1) + +VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress) +VALUE_OPCODE(ImageQueryLod, U32x4, ImageResource, SamplerResource, ImageAddress) +VALUE_OPCODE(ImageRead, U32x4, ImageResource, ImageAddress, U1) +VALUE_OPCODE(ImageWrite, Void, ImageResource, ImageAddress, U32x4, U1) +VALUE_OPCODE(ImageSampleRaw, U32x4, ImageResource, SamplerResource, ImageAddress) +VALUE_OPCODE(ImageGatherRaw, U32x4, ImageResource, SamplerResource, ImageAddress) +VALUE_OPCODE(ImageAtomicIAdd32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(GetAttribute, U32, U32, U32) +VALUE_OPCODE(SetAttribute, Void, U32x4, U1) + +VALUE_OPCODE(ControlNop, Void) +VALUE_OPCODE(Waitcnt, Void) +VALUE_OPCODE(Barrier, Void) +VALUE_OPCODE(Sendmsg, Void) +VALUE_OPCODE(TtraceData, Void) +VALUE_OPCODE(InstPrefetch, Void) +VALUE_OPCODE(SelectU32, U32, U1, U32, U32) diff --git a/src/graphics/shader/recompiler/ir/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp similarity index 70% rename from src/graphics/shader/recompiler/ir/BindingLayout.cpp rename to src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index 0667a13fb..d16687ef8 100644 --- a/src/graphics/shader/recompiler/ir/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -1,6 +1,6 @@ -#include "graphics/shader/recompiler/ir/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include #include @@ -88,105 +88,21 @@ bool ImageBinding(const ImageResource& image, DescriptorBindingKind& result) { } } -bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector& visited, - std::set& registers) { - if (id <= ScalarProvenance::Unknown) { - return true; - } - if (id >= provenance.values.size()) { - return false; - } - if (visited[id] != 0) { - return true; - } - visited[id] = 1; - const auto& value = provenance.values[id]; - if (value.op == ScalarValueOp::UserData) { - registers.insert(value.imm); - return true; - } - if (value.op == ScalarValueOp::Phi) { - for (const auto arg: value.phi_args) { - if (!CollectValue(provenance, arg, visited, registers)) { - return false; - } - } - return true; - } - for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) { - if (!CollectValue(provenance, value.args[i], visited, registers)) { - return false; - } - } - return true; -} - -bool CollectSource(const Program& program, uint32_t source, bool allow_unknown, - std::vector& visited, std::set& registers) { - if (allow_unknown && source == ScalarProvenance::Unknown) { - return true; - } - if (source < 2u || source - 2u >= program.provenance.descriptors.size()) { - return false; - } - const auto& descriptor = program.provenance.descriptors[source - 2u]; - for (uint32_t i = 0; i < descriptor.dword_count; i++) { - if (!CollectValue(program.provenance, descriptor.dwords[i], visited, registers)) { - return false; - } - } - return true; -} - bool CollectUserData(const Program& program, std::vector& result) { - std::set registers; - std::vector visited(program.provenance.values.size()); - for (const auto& buffer: program.info.buffers) { - if (!CollectSource(program, buffer.source, false, visited, registers)) { - return false; - } - } - for (const auto& image: program.info.images) { - if (!CollectSource(program, image.source, false, visited, registers)) { - return false; - } - } - for (const auto& sampler: program.info.samplers) { - if (!CollectSource(program, sampler.source, false, visited, registers)) { - return false; - } - } - for (const auto& address: program.info.addresses) { - if (!CollectSource(program, address.source, true, visited, registers)) { - return false; - } - } - for (const auto& read: program.srt.reads) { - if (!CollectValue(program.provenance, read.value, visited, registers)) { - return false; - } - } - for (const auto read: program.srt.dynamic_reads) { - if (!CollectValue(program.provenance, read, visited, registers)) { - return false; - } - } - for (const auto source: program.srt.dynamic_sources) { - if (!CollectSource(program, source, true, visited, registers)) { - return false; - } - } - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == Opcode::SLoadDword && inst.memory.kind == ResourceKind::ScalarBuffer && - !CollectValue(program.provenance, inst.scalar_value, visited, registers)) { + std::set registers; + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) { + continue; + } + if (inst.Arg(0).GetType() != Type::ScalarReg) { return false; } - for (uint32_t i = 0; i < inst.src_count; i++) { - if (!CollectValue(program.provenance, inst.scalar_sources[i], visited, registers)) { - return false; - } + const auto index = RegIndex(inst.Arg(0).ScalarRegister()); + if (index >= NumScalarRegs) { + return false; } + registers.insert(index); } } result.assign(registers.begin(), registers.end()); @@ -200,11 +116,16 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind, } bool UsesGds(const Program& program) { - for (const auto& block: program.blocks) { - for (const auto& inst: block.instructions) { - if (inst.memory.kind == ResourceKind::Gds) { + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() == ValueOpcode::GetGdsResource && inst.HasUses()) { return true; } + for (size_t index = 0; index < inst.NumArgs(); index++) { + if (inst.Arg(index).GetType() == Type::GdsResource) { + return true; + } + } } } return false; @@ -232,13 +153,19 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std } return false; } + if (program.values == nullptr) { + if (error != nullptr) { + *error = "typed value program is not available"; + } + return false; + } BindingLayout next; next.descriptor_set = options.descriptor_set; next.push_constant_offset = options.push_constant_offset; if (!CollectUserData(program, next.user_data_registers)) { if (error != nullptr) { - *error = "shader plan contains an invalid scalar provenance reference"; + *error = "typed shader contains an invalid user-data register"; } return false; } @@ -294,7 +221,7 @@ bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std } AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources)); } - if (!program.srt.reads.empty()) { + if (!program.values->srt_reads.empty()) { AddBinding(next, DescriptorBindingKind::FlattenedSrt); } diff --git a/src/graphics/shader/recompiler/ir/BindingLayout.h b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h similarity index 100% rename from src/graphics/shader/recompiler/ir/BindingLayout.h rename to src/graphics/shader/recompiler/ir/passes/BindingLayout.h diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp new file mode 100644 index 000000000..a1cb4c3d0 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp @@ -0,0 +1,612 @@ +#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +Value Arg(const Inst& inst, size_t index) { + return inst.Arg(index).Resolve(); +} + +bool IsImmediate(Value value, Type type) { + return value.IsImmediate() && value.GetType() == type; +} + +void Replace(Inst& inst, Value value) { + inst.ReplaceUsesWith(value.Resolve()); +} + +template +bool FoldU32(Inst& inst, Function function) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (!IsImmediate(lhs, Type::U32) || !IsImmediate(rhs, Type::U32)) { + return false; + } + Replace(inst, Value(static_cast(function(lhs.U32(), rhs.U32())))); + return true; +} + +template +bool FoldU64(Inst& inst, Function function) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (!IsImmediate(lhs, Type::U64) || !IsImmediate(rhs, Type::U64)) { + return false; + } + Replace(inst, Value(static_cast(function(lhs.U64(), rhs.U64())))); + return true; +} + +template +bool FoldU64Shift(Inst& inst, Function function) { + const auto value = Arg(inst, 0); + const auto shift = Arg(inst, 1); + if (!IsImmediate(value, Type::U64) || !IsImmediate(shift, Type::U32)) { + return false; + } + Replace(inst, Value(static_cast(function(value.U64(), shift.U32())))); + return true; +} + +template +bool FoldU32Compare(Inst& inst, Function function) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (!IsImmediate(lhs, Type::U32) || !IsImmediate(rhs, Type::U32)) { + return false; + } + Replace(inst, Value(function(lhs.U32(), rhs.U32()))); + return true; +} + +template +bool FoldU64Compare(Inst& inst, Function function) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (!IsImmediate(lhs, Type::U64) || !IsImmediate(rhs, Type::U64)) { + return false; + } + Replace(inst, Value(function(lhs.U64(), rhs.U64()))); + return true; +} + +template +bool FoldLogical(Inst& inst, Function function) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (!IsImmediate(lhs, Type::U1) || !IsImmediate(rhs, Type::U1)) { + return false; + } + Replace(inst, Value(function(lhs.U1(), rhs.U1()))); + return true; +} + +bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (IsImmediate(lhs, type) && + (type == Type::U32 ? lhs.U32() == identity : lhs.U64() == identity)) { + Replace(inst, rhs); + return true; + } + if (IsImmediate(rhs, type) && + (type == Type::U32 ? rhs.U32() == identity : rhs.U64() == identity)) { + Replace(inst, lhs); + return true; + } + return false; +} + +bool FoldSelect(Inst& inst) { + const auto condition = Arg(inst, 0); + const auto true_value = Arg(inst, 1); + const auto false_value = Arg(inst, 2); + if (IsImmediate(condition, Type::U1)) { + Replace(inst, condition.U1() ? true_value : false_value); + return true; + } + if (true_value == false_value) { + Replace(inst, true_value); + return true; + } + return false; +} + +bool FoldPhi(Inst& inst) { + Value same; + for (size_t index = 0; index < inst.NumArgs(); index++) { + const auto value = Arg(inst, index); + if (value.TryInstruction() == &inst) { + continue; + } + if (same.IsEmpty()) { + same = value; + } else if (same != value) { + return false; + } + } + if (same.IsEmpty()) { + return false; + } + Replace(inst, same); + return true; +} + +bool FoldBitCast(Inst& inst, ValueOpcode reverse) { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::F32) && inst.GetOpcode() == ValueOpcode::BitCastU32F32) { + Replace(inst, Value(std::bit_cast(value.F32Value()))); + return true; + } + if (IsImmediate(value, Type::U32) && inst.GetOpcode() == ValueOpcode::BitCastF32U32) { + Replace(inst, Value::F32(std::bit_cast(value.U32()))); + return true; + } + if (IsImmediate(value, Type::F16) && inst.GetOpcode() == ValueOpcode::BitCastU16F16) { + Replace(inst, Value(value.F16Bits())); + return true; + } + if (IsImmediate(value, Type::U16) && inst.GetOpcode() == ValueOpcode::BitCastF16U16) { + Replace(inst, Value::F16(value.U16())); + return true; + } + if (auto* producer = value.TryInstruction(); + producer != nullptr && producer->GetOpcode() == reverse) { + Replace(inst, producer->Arg(0)); + return true; + } + return false; +} + +bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components) { + const auto composite = Arg(inst, 0); + const auto index = Arg(inst, 1); + if (!IsImmediate(index, Type::U32) || index.U32() >= components) { + return false; + } + const auto component = index.U32(); + const auto* producer = composite.TryInstruction(); + if (producer != nullptr && producer->GetOpcode() == construct) { + Replace(inst, producer->Arg(component)); + return true; + } + if (component < 2u && producer != nullptr && + producer->GetOpcode() == ValueOpcode::IAddCarry32) { + const auto lhs = producer->Arg(0).Resolve(); + const auto rhs = producer->Arg(1).Resolve(); + if (IsImmediate(lhs, Type::U32) && IsImmediate(rhs, Type::U32)) { + const auto sum = static_cast(lhs.U32()) + rhs.U32(); + Replace(inst, Value(component == 0u ? static_cast(sum) + : static_cast(sum >> 32u))); + return true; + } + } + if (component < 2u && producer != nullptr && + producer->GetOpcode() == ValueOpcode::UnpackUint2x32) { + const auto packed = producer->Arg(0).Resolve(); + if (IsImmediate(packed, Type::U64)) { + Replace(inst, Value(static_cast(packed.U64() >> (component * 32u)))); + return true; + } + } + return false; +} + +void FoldInstruction(Inst& inst) { + switch (inst.GetOpcode()) { + case ValueOpcode::Phi: FoldPhi(inst); return; + case ValueOpcode::SelectU1: + case ValueOpcode::SelectU32: + case ValueOpcode::SelectF32: FoldSelect(inst); return; + case ValueOpcode::BitFieldInsert: { + const auto base = Arg(inst, 0); + const auto insert = Arg(inst, 1); + const auto offset = Arg(inst, 2); + const auto count = Arg(inst, 3); + if (IsImmediate(base, Type::U32) && IsImmediate(insert, Type::U32) && + IsImmediate(offset, Type::U32) && IsImmediate(count, Type::U32) && + offset.U32() <= 32u && count.U32() <= 32u - offset.U32()) { + if (count.U32() == 0u) { + Replace(inst, base); + return; + } + const auto mask = count.U32() == 32u + ? UINT32_MAX + : ((uint32_t {1} << count.U32()) - 1u) << offset.U32(); + Replace(inst, + Value((base.U32() & ~mask) | ((insert.U32() << offset.U32()) & mask))); + } + return; + } + case ValueOpcode::BitFieldUExtract: + case ValueOpcode::BitFieldSExtract: { + const auto value = Arg(inst, 0); + const auto offset = Arg(inst, 1); + const auto count = Arg(inst, 2); + if (!IsImmediate(value, Type::U32) || !IsImmediate(offset, Type::U32) || + !IsImmediate(count, Type::U32) || offset.U32() > 32u || + count.U32() > 32u - offset.U32()) { + return; + } + if (count.U32() == 0u) { + Replace(inst, Value(0u)); + } else if (inst.GetOpcode() == ValueOpcode::BitFieldUExtract) { + const auto mask = + count.U32() == 32u ? UINT32_MAX : (uint32_t {1} << count.U32()) - 1u; + Replace(inst, Value((value.U32() >> offset.U32()) & mask)); + } else { + const auto left = 32u - offset.U32() - count.U32(); + const auto bits = value.U32() << left; + Replace(inst, Value(static_cast(std::bit_cast(bits) >> + (left + offset.U32())))); + } + return; + } + case ValueOpcode::BitCastU16F16: FoldBitCast(inst, ValueOpcode::BitCastF16U16); return; + case ValueOpcode::BitCastF16U16: FoldBitCast(inst, ValueOpcode::BitCastU16F16); return; + case ValueOpcode::BitCastU32F32: FoldBitCast(inst, ValueOpcode::BitCastF32U32); return; + case ValueOpcode::BitCastF32U32: FoldBitCast(inst, ValueOpcode::BitCastU32F32); return; + case ValueOpcode::ConvertU16U32: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U32)) { + Replace(inst, Value(static_cast(value.U32()))); + } + return; + } + case ValueOpcode::ConvertU32U16: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U16)) { + Replace(inst, Value(static_cast(value.U16()))); + } else if (auto* producer = value.TryInstruction(); + producer != nullptr && producer->GetOpcode() == ValueOpcode::ConvertU16U32) { + Replace(inst, producer->Arg(0)); + } + return; + } + case ValueOpcode::ConvertU8U32: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U32)) { + Replace(inst, Value(static_cast(value.U32()))); + } + return; + } + case ValueOpcode::ConvertU32U8: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U8)) { + Replace(inst, Value(static_cast(value.U8()))); + } else if (auto* producer = value.TryInstruction(); + producer != nullptr && producer->GetOpcode() == ValueOpcode::ConvertU8U32) { + Replace(inst, producer->Arg(0)); + } + return; + } + case ValueOpcode::CompositeExtractU32x2: + FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2); + return; + case ValueOpcode::CompositeExtractU32x4: + FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4); + return; + case ValueOpcode::PackUint2x32: { + const auto value = Arg(inst, 0); + const auto* producer = value.TryInstruction(); + if (producer != nullptr && producer->GetOpcode() == ValueOpcode::UnpackUint2x32) { + Replace(inst, producer->Arg(0)); + } else if (producer != nullptr && + producer->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { + const auto low = producer->Arg(0).Resolve(); + const auto high = producer->Arg(1).Resolve(); + if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) { + Replace(inst, Value(static_cast(low.U32()) | + (static_cast(high.U32()) << 32u))); + } + } + return; + } + case ValueOpcode::UnpackUint2x32: { + const auto value = Arg(inst, 0); + const auto* producer = value.TryInstruction(); + if (producer != nullptr && producer->GetOpcode() == ValueOpcode::PackUint2x32) { + Replace(inst, producer->Arg(0)); + } + return; + } + case ValueOpcode::IAdd32: + if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a + b; })) { + ReplaceBinaryIdentity(inst, Type::U32, 0u); + } + return; + case ValueOpcode::IAdd64: + if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a + b; })) { + ReplaceBinaryIdentity(inst, Type::U64, 0u); + } + return; + case ValueOpcode::ISub32: { + if (FoldU32(inst, [](uint32_t a, uint32_t b) { return a - b; })) { + return; + } + const auto rhs = Arg(inst, 1); + if (IsImmediate(rhs, Type::U32) && rhs.U32() == 0u) { + Replace(inst, Arg(inst, 0)); + } + return; + } + case ValueOpcode::ISub64: { + if (FoldU64(inst, [](uint64_t a, uint64_t b) { return a - b; })) { + return; + } + const auto rhs = Arg(inst, 1); + if (IsImmediate(rhs, Type::U64) && rhs.U64() == 0u) { + Replace(inst, Arg(inst, 0)); + } + return; + } + case ValueOpcode::IMul32: + if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a * b; })) { + ReplaceBinaryIdentity(inst, Type::U32, 1u); + } + return; + case ValueOpcode::IMul64: + if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a * b; })) { + ReplaceBinaryIdentity(inst, Type::U64, 1u); + } + return; + case ValueOpcode::UDiv32: { + const auto rhs = Arg(inst, 1); + if (IsImmediate(rhs, Type::U32) && rhs.U32() == 1u) { + Replace(inst, Arg(inst, 0)); + } else if (IsImmediate(rhs, Type::U32) && rhs.U32() != 0u) { + FoldU32(inst, [](uint32_t a, uint32_t b) { return a / b; }); + } + return; + } + case ValueOpcode::SMulHi: + FoldU32(inst, [](uint32_t a, uint32_t b) { + const auto product = static_cast(std::bit_cast(a)) * + static_cast(std::bit_cast(b)); + return static_cast(static_cast(product) >> 32u); + }); + return; + case ValueOpcode::UMulHi: + FoldU32(inst, [](uint32_t a, uint32_t b) { + return static_cast((static_cast(a) * b) >> 32u); + }); + return; + case ValueOpcode::IAbs32: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U32)) { + Replace(inst, + Value((value.U32() & 0x80000000u) != 0u ? 0u - value.U32() : value.U32())); + } + return; + } + case ValueOpcode::ShiftLeftLogical32: + if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a << (b & 31u); })) { + const auto shift = Arg(inst, 1); + if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) { + Replace(inst, Arg(inst, 0)); + } + } + return; + case ValueOpcode::ShiftRightLogical32: + if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a >> (b & 31u); })) { + const auto shift = Arg(inst, 1); + if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) { + Replace(inst, Arg(inst, 0)); + } + } + return; + case ValueOpcode::ShiftRightArithmetic32: + FoldU32(inst, [](uint32_t a, uint32_t b) { + return static_cast(std::bit_cast(a) >> (b & 31u)); + }); + return; + case ValueOpcode::ShiftLeftLogical64: + if (!FoldU64Shift(inst, [](uint64_t a, uint32_t b) { return a << (b & 63u); })) { + const auto shift = Arg(inst, 1); + if (IsImmediate(shift, Type::U32) && (shift.U32() & 63u) == 0u) { + Replace(inst, Arg(inst, 0)); + } + } + return; + case ValueOpcode::ShiftRightLogical64: + if (!FoldU64Shift(inst, [](uint64_t a, uint32_t b) { return a >> (b & 63u); })) { + const auto shift = Arg(inst, 1); + if (IsImmediate(shift, Type::U32) && (shift.U32() & 63u) == 0u) { + Replace(inst, Arg(inst, 0)); + } + } + return; + case ValueOpcode::ShiftRightArithmetic64: + FoldU64Shift(inst, [](uint64_t a, uint32_t b) { + return static_cast(std::bit_cast(a) >> (b & 63u)); + }); + return; + case ValueOpcode::BitwiseAnd32: + if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a & b; })) { + ReplaceBinaryIdentity(inst, Type::U32, 0xffffffffu); + } + return; + case ValueOpcode::BitwiseAnd64: + if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a & b; })) { + ReplaceBinaryIdentity(inst, Type::U64, UINT64_MAX); + } + return; + case ValueOpcode::BitwiseOr32: + case ValueOpcode::BitwiseXor32: + if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) { + return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b; + })) { + ReplaceBinaryIdentity(inst, Type::U32, 0u); + } + return; + case ValueOpcode::BitwiseOr64: + if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a | b; })) { + ReplaceBinaryIdentity(inst, Type::U64, 0u); + } + return; + case ValueOpcode::BitwiseNot32: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U32)) { + Replace(inst, Value(~value.U32())); + } else if (auto* producer = value.TryInstruction(); + producer != nullptr && producer->GetOpcode() == ValueOpcode::BitwiseNot32) { + Replace(inst, producer->Arg(0)); + } + return; + } + case ValueOpcode::BitCount32: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U32)) { + Replace(inst, Value(static_cast(std::popcount(value.U32())))); + } + return; + } + case ValueOpcode::BitCount64: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U64)) { + Replace(inst, Value(static_cast(std::popcount(value.U64())))); + } + return; + } + case ValueOpcode::SMin32: + case ValueOpcode::SMax32: + FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) { + const auto lhs = std::bit_cast(a); + const auto rhs = std::bit_cast(b); + return opcode == ValueOpcode::SMin32 ? (lhs < rhs ? a : b) : (lhs > rhs ? a : b); + }); + return; + case ValueOpcode::UMin32: + FoldU32(inst, [](uint32_t a, uint32_t b) { return std::min(a, b); }); + return; + case ValueOpcode::UMax32: + FoldU32(inst, [](uint32_t a, uint32_t b) { return std::max(a, b); }); + return; + case ValueOpcode::IEqual32: + FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a == b; }); + return; + case ValueOpcode::INotEqual32: + FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a != b; }); + return; + case ValueOpcode::ULessThan32: + FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a < b; }); + return; + case ValueOpcode::ULessThanEqual32: + FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a <= b; }); + return; + case ValueOpcode::UGreaterThan32: + FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a > b; }); + return; + case ValueOpcode::UGreaterThanEqual32: + FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a >= b; }); + return; + case ValueOpcode::SLessThan32: + case ValueOpcode::SLessThanEqual32: + case ValueOpcode::SGreaterThan32: + case ValueOpcode::SGreaterThanEqual32: + FoldU32Compare(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) { + const auto lhs = std::bit_cast(a); + const auto rhs = std::bit_cast(b); + switch (opcode) { + case ValueOpcode::SLessThan32: return lhs < rhs; + case ValueOpcode::SLessThanEqual32: return lhs <= rhs; + case ValueOpcode::SGreaterThan32: return lhs > rhs; + default: return lhs >= rhs; + } + }); + return; + case ValueOpcode::IEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a == b; }); + return; + case ValueOpcode::INotEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a != b; }); + return; + case ValueOpcode::ULessThan64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a < b; }); + return; + case ValueOpcode::ULessThanEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; }); + return; + case ValueOpcode::UGreaterThan64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; }); + return; + case ValueOpcode::UGreaterThanEqual64: + FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; }); + return; + case ValueOpcode::SLessThan64: + case ValueOpcode::SLessThanEqual64: + case ValueOpcode::SGreaterThan64: + case ValueOpcode::SGreaterThanEqual64: + FoldU64Compare(inst, [opcode = inst.GetOpcode()](uint64_t a, uint64_t b) { + const auto lhs = std::bit_cast(a); + const auto rhs = std::bit_cast(b); + switch (opcode) { + case ValueOpcode::SLessThan64: return lhs < rhs; + case ValueOpcode::SLessThanEqual64: return lhs <= rhs; + case ValueOpcode::SGreaterThan64: return lhs > rhs; + default: return lhs >= rhs; + } + }); + return; + case ValueOpcode::LogicalAnd: + if (!FoldLogical(inst, [](bool a, bool b) { return a && b; })) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (IsImmediate(lhs, Type::U1)) { + Replace(inst, lhs.U1() ? rhs : lhs); + } else if (IsImmediate(rhs, Type::U1)) { + Replace(inst, rhs.U1() ? lhs : rhs); + } + } + return; + case ValueOpcode::LogicalOr: + if (!FoldLogical(inst, [](bool a, bool b) { return a || b; })) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (IsImmediate(lhs, Type::U1)) { + Replace(inst, lhs.U1() ? lhs : rhs); + } else if (IsImmediate(rhs, Type::U1)) { + Replace(inst, rhs.U1() ? rhs : lhs); + } + } + return; + case ValueOpcode::LogicalXor: + if (!FoldLogical(inst, [](bool a, bool b) { return a != b; })) { + const auto lhs = Arg(inst, 0); + const auto rhs = Arg(inst, 1); + if (IsImmediate(lhs, Type::U1) && !lhs.U1()) { + Replace(inst, rhs); + } else if (IsImmediate(rhs, Type::U1) && !rhs.U1()) { + Replace(inst, lhs); + } + } + return; + case ValueOpcode::LogicalNot: { + const auto value = Arg(inst, 0); + if (IsImmediate(value, Type::U1)) { + Replace(inst, Value(!value.U1())); + } else if (auto* producer = value.TryInstruction(); + producer != nullptr && producer->GetOpcode() == ValueOpcode::LogicalNot) { + Replace(inst, producer->Arg(0)); + } + return; + } + default: return; + } +} + +} // namespace + +void ConstantPropagationPass(const BlockList& blocks) { + for (auto* block: blocks) { + for (auto& inst: block->Instructions()) { + FoldInstruction(inst); + } + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h new file mode 100644 index 000000000..e158c1ea1 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/ConstantPropagation.h @@ -0,0 +1,9 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Block.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +void ConstantPropagationPass(const BlockList& blocks); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp b/src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp new file mode 100644 index 000000000..3ef1f3589 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp @@ -0,0 +1,40 @@ +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +void RemoveIdentities(const BlockList& blocks) { + for (auto* block: blocks) { + auto& instructions = block->Instructions(); + for (auto inst = instructions.begin(); inst != instructions.end();) { + if (inst->GetOpcode() != ValueOpcode::Identity) { + inst++; + continue; + } + const auto replacement = inst->Arg(0); + inst->ReplaceUsesWith(replacement, false); + inst = instructions.erase(inst); + } + } +} + +void EliminateDeadCode(const BlockList& blocks) { + bool changed; + do { + changed = false; + for (auto block = blocks.rbegin(); block != blocks.rend(); block++) { + auto& instructions = (*block)->Instructions(); + auto inst = instructions.end(); + while (inst != instructions.begin()) { + --inst; + if (inst->HasUses() || inst->MayHaveSideEffects()) { + continue; + } + inst->Invalidate(); + inst = instructions.erase(inst); + changed = true; + } + } + } while (changed); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.h b/src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.h new file mode 100644 index 000000000..6d2edafb5 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.h @@ -0,0 +1,10 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Block.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +void RemoveIdentities(const BlockList& blocks); +void EliminateDeadCode(const BlockList& blocks); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp new file mode 100644 index 000000000..d9fa76921 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp @@ -0,0 +1,141 @@ +#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" + +#include + +#include + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +struct ChainResult { + Value value; + Inst* write = nullptr; +}; + +ChainResult SearchChain(Value value, uint32_t lane, uint32_t wave_size) { + for (;;) { + value = value.Resolve(); + auto* inst = value.TryInstruction(); + if (inst == nullptr || inst->GetOpcode() != ValueOpcode::WriteLane) { + return {value}; + } + const auto selector = inst->Arg(2).Resolve(); + if (!selector.IsImmediate() || selector.GetType() != Type::U32) { + return {value}; + } + if (selector.U32() % wave_size == lane) { + return {value, inst}; + } + value = inst->Arg(0); + } +} + +bool IsPossibleToEliminate(Value source, uint32_t lane, uint32_t wave_size) { + std::queue queue; + std::unordered_set visited; + queue.push(source); + + while (!queue.empty()) { + const auto chain = SearchChain(queue.front(), lane, wave_size); + queue.pop(); + if (chain.write != nullptr) { + continue; + } + auto* inst = chain.value.TryInstruction(); + if (inst == nullptr || inst->GetOpcode() != ValueOpcode::Phi || inst->NumArgs() == 0) { + return false; + } + if (!visited.insert(inst).second) { + continue; + } + for (size_t index = inst->NumArgs(); index-- > 0;) { + queue.push(inst->Arg(index)); + } + } + return true; +} + +using PhiMap = std::unordered_map; + +Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_size) { + const auto chain = SearchChain(source, lane, wave_size); + if (chain.write != nullptr) { + return chain.write->Arg(1); + } + + auto* inst = chain.value.ResolveInstruction(); + EXIT_IF(inst->GetOpcode() != ValueOpcode::Phi); + const auto [entry, is_new] = phi_map.try_emplace(inst); + if (!is_new) { + return Value(entry->second); + } + + auto* block = inst->Parent(); + auto insertion_point = std::find_if(block->begin(), block->end(), + [&](const Inst& candidate) { return &candidate == inst; }); + EXIT_IF(insertion_point == block->end()); + auto& phi = *block->PrependNewInst(insertion_point, ValueOpcode::Phi); + phi.SetFlags(Type::U32); + entry->second = φ + + std::vector arguments; + arguments.reserve(inst->NumArgs()); + for (size_t index = 0; index < inst->NumArgs(); index++) { + arguments.push_back(GetRealValue(phi_map, inst->Arg(index), lane, wave_size)); + } + const auto first = arguments.front().Resolve(); + if (std::ranges::all_of(arguments, + [&](Value argument) { return argument.Resolve() == first; })) { + phi.ReplaceUsesWith(first); + } else { + for (size_t index = 0; index < arguments.size(); index++) { + phi.AddPhiOperand(inst->PhiBlock(index), arguments[index]); + } + } + return Value(&phi); +} + +} // namespace + +ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size) { + ReadLaneStats stats; + if (wave_size != 32u && wave_size != 64u) { + return stats; + } + + for (auto* block: program.blocks) { + for (auto& inst: *block) { + if (inst.GetOpcode() != ValueOpcode::ReadLane) { + continue; + } + const auto selector = inst.Arg(1).Resolve(); + if (!selector.IsImmediate() || selector.GetType() != Type::U32) { + continue; + } + + const auto lane = selector.U32() % wave_size; + const auto chain = SearchChain(inst.Arg(0), lane, wave_size); + if (chain.write != nullptr) { + inst.ReplaceUsesWith(chain.write->Arg(1)); + stats.rewritten_reads++; + continue; + } + auto* producer = chain.value.TryInstruction(); + if (producer == nullptr || producer->GetOpcode() != ValueOpcode::Phi || + !IsPossibleToEliminate(chain.value, lane, wave_size)) { + continue; + } + + PhiMap phi_map; + inst.ReplaceUsesWith(GetRealValue(phi_map, chain.value, lane, wave_size)); + stats.rewritten_reads++; + } + } + return stats; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h new file mode 100644 index 000000000..487a43a23 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/ReadLaneElimination.h @@ -0,0 +1,13 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +struct ReadLaneStats { + uint32_t rewritten_reads = 0; +}; + +[[nodiscard]] ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp similarity index 89% rename from src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp rename to src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 80ac6b9ab..b056e2864 100644 --- a/src/graphics/shader/recompiler/ir/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -1,6 +1,7 @@ -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/guest_gpu/gpu_format.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/shaderBindings.h" #include @@ -81,11 +82,29 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou uint64_t AddressSpecialization(const AddressResource& resource, const ResourceSnapshot::Address& snapshot) { - return resource.kind == ResourceKind::Flat || resource.source == ScalarProvenance::Unknown + return resource.kind == ResourceKind::Flat || resource.unbased ? snapshot.binding_base : snapshot.guest_base - snapshot.binding_base; } +bool IsImageOp(ValueOpcode op) { + switch (op) { + case ValueOpcode::ImageQueryDimensions: + case ValueOpcode::ImageQueryLod: + case ValueOpcode::ImageRead: + case ValueOpcode::ImageWrite: + case ValueOpcode::ImageSampleRaw: + case ValueOpcode::ImageGatherRaw: + case ValueOpcode::ImageAtomicIAdd32: + case ValueOpcode::ImageAtomicUMin32: + case ValueOpcode::ImageAtomicUMax32: + case ValueOpcode::ImageAtomicAnd32: + case ValueOpcode::ImageAtomicOr32: + case ValueOpcode::ImageAtomicXor32: return true; + default: return false; + } +} + } // namespace bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot, @@ -105,7 +124,8 @@ bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& sn } return false; } - if (snapshot.flattened_srt.size() != program.srt.reads.size()) { + if (program.values == nullptr || + snapshot.flattened_srt.size() != program.values->srt_reads.size()) { if (error != nullptr) { *error = "flattened SRT snapshot does not match the shader plan"; } @@ -280,7 +300,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, requests.push_back({sampler.source, sampler.first_use_pc}); } for (const auto& address: program.info.addresses) { - if (address.source != ScalarProvenance::Unknown) { + if (!address.unbased) { requests.push_back({address.source, address.first_use_pc}); } } @@ -311,7 +331,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime, next.samplers.assign(cursor, cursor + program.info.samplers.size()); cursor += program.info.samplers.size(); for (const auto& address: program.info.addresses) { - if (address.source != ScalarProvenance::Unknown) { + if (!address.unbased) { const auto value = *cursor++; auto base = (static_cast(value.dwords[0]) | static_cast(value.dwords[1]) << 32u) & @@ -430,38 +450,42 @@ bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std } } } - struct ImagePatch { - std::reference_wrapper inst; - ResourceKind kind; - Decoder::ImageDimension dimension; - bool cube; - }; - std::vector patches; - for (auto& block: program.blocks) { - for (auto& inst: block.instructions) { - if (inst.memory.kind != ResourceKind::Image && - inst.memory.kind != ResourceKind::ImageUint && - inst.memory.kind != ResourceKind::StorageImage && - inst.memory.kind != ResourceKind::StorageImageUint) { + if (program.values == nullptr) { + if (error != nullptr) { + *error = "typed SSA is not ready"; + } + return false; + } + auto memory_info = program.values->memory_info; + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (!IsImageOp(inst.GetOpcode())) { continue; } - if (inst.memory.resource >= next.images.size()) { + const auto index = inst.Flags().index; + if (index >= memory_info.size()) { if (error != nullptr) { - *error = fmt::format("image instruction at pc 0x{:08x} has invalid resource {}", - inst.pc, inst.memory.resource); + *error = fmt::format("typed image instruction has invalid memory metadata {}", + index); } return false; } - const auto& image = next.images[inst.memory.resource]; - patches.push_back({std::ref(inst), image.kind, image.dimension, image.cube}); + auto& memory = memory_info[index]; + if (memory.resource >= next.images.size()) { + if (error != nullptr) { + *error = fmt::format("typed image instruction has invalid resource {}", + memory.resource); + } + return false; + } + const auto& image = next.images[memory.resource]; + memory.kind = image.kind; + memory.image_dimension = image.dimension; + memory.image_cube = image.cube; } } - program.info = std::move(next); - for (const auto& patch: patches) { - patch.inst.get().memory.kind = patch.kind; - patch.inst.get().memory.image_dimension = patch.dimension; - patch.inst.get().memory.image_cube = patch.cube; - } + program.info = std::move(next); + program.values->memory_info = std::move(memory_info); return true; } diff --git a/src/graphics/shader/recompiler/ir/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h similarity index 96% rename from src/graphics/shader/recompiler/ir/ResourceMaterialization.h rename to src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index 0023b6f53..af73f31b0 100644 --- a/src/graphics/shader/recompiler/ir/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -1,7 +1,7 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ -#include "graphics/shader/recompiler/ir/SrtWalker.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" namespace Libs::Graphics::ShaderRecompiler::IR { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp new file mode 100644 index 000000000..d44e973e5 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -0,0 +1,677 @@ +#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" + +#include "graphics/shader/recompiler/ir/ValueProgram.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u); + +const char* StageName(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "vertex"; + case ShaderType::Pixel: return "pixel"; + case ShaderType::Fetch: return "fetch"; + case ShaderType::Compute: return "compute"; + default: return "unknown"; + } +} + +bool IsBufferAtomic(ValueOpcode op) { + switch (op) { + case ValueOpcode::BufferAtomicSwap32: + case ValueOpcode::BufferAtomicIAdd32: + case ValueOpcode::BufferAtomicISub32: + case ValueOpcode::BufferAtomicSMin32: + case ValueOpcode::BufferAtomicUMin32: + case ValueOpcode::BufferAtomicSMax32: + case ValueOpcode::BufferAtomicUMax32: + case ValueOpcode::BufferAtomicAnd32: + case ValueOpcode::BufferAtomicOr32: + case ValueOpcode::BufferAtomicXor32: + case ValueOpcode::BufferAtomicFMin32: + case ValueOpcode::BufferAtomicFMax32: return true; + default: return false; + } +} + +bool IsBufferStore(ValueOpcode op) { + switch (op) { + case ValueOpcode::StoreBufferU8: + case ValueOpcode::StoreBufferU16: + case ValueOpcode::StoreBufferU32: return true; + default: return IsBufferAtomic(op); + } +} + +bool IsBuffer(ValueOpcode op) { + switch (op) { + case ValueOpcode::ReadConstBuffer: + case ValueOpcode::LoadBufferU8: + case ValueOpcode::LoadBufferU16: + case ValueOpcode::LoadBufferU32: + case ValueOpcode::StoreBufferU8: + case ValueOpcode::StoreBufferU16: + case ValueOpcode::StoreBufferU32: return true; + default: return IsBufferAtomic(op); + } +} + +bool IsAddressStore(ValueOpcode op) { + switch (op) { + case ValueOpcode::StoreAddressU8: + case ValueOpcode::StoreAddressU16: + case ValueOpcode::StoreAddressU32: return true; + default: return false; + } +} + +bool IsAddress(ValueOpcode op) { + switch (op) { + case ValueOpcode::LoadAddressU8: + case ValueOpcode::LoadAddressU16: + case ValueOpcode::LoadAddressU32: + case ValueOpcode::StoreAddressU8: + case ValueOpcode::StoreAddressU16: + case ValueOpcode::StoreAddressU32: return true; + default: return false; + } +} + +bool IsImageAtomic(ValueOpcode op) { + switch (op) { + case ValueOpcode::ImageAtomicIAdd32: + case ValueOpcode::ImageAtomicUMin32: + case ValueOpcode::ImageAtomicUMax32: + case ValueOpcode::ImageAtomicAnd32: + case ValueOpcode::ImageAtomicOr32: + case ValueOpcode::ImageAtomicXor32: return true; + default: return false; + } +} + +bool IsImageStore(ValueOpcode op) { + return op == ValueOpcode::ImageWrite || IsImageAtomic(op); +} + +bool IsImage(ValueOpcode op) { + switch (op) { + case ValueOpcode::ImageQueryDimensions: + case ValueOpcode::ImageQueryLod: + case ValueOpcode::ImageRead: + case ValueOpcode::ImageWrite: + case ValueOpcode::ImageSampleRaw: + case ValueOpcode::ImageGatherRaw: return true; + default: return IsImageAtomic(op); + } +} + +bool NeedsSampler(ValueOpcode op) { + return op == ValueOpcode::ImageQueryLod || op == ValueOpcode::ImageSampleRaw || + op == ValueOpcode::ImageGatherRaw; +} + +uint32_t ByteExtent(const MemoryInfo& memory) { + const auto bytes = std::max((memory.data_bits + 7u) / 8u, 1u); + const auto count = std::max(memory.data_dwords, 1u); + const auto end = static_cast(memory.offset) + static_cast(bytes) * count; + return end > UINT32_MAX ? UINT32_MAX : static_cast(end); +} + +ImageMipMode MipMode(const MemoryInfo& memory) { + const bool storage = + memory.kind == ResourceKind::StorageImage || memory.kind == ResourceKind::StorageImageUint; + return storage && memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None; +} + +class Tracker { +public: + Tracker(Program& program, ValueProgram& values) + : m_program(program), m_values(values), m_info(program.info) { + m_info.buffers.clear(); + m_info.addresses.clear(); + m_info.images.clear(); + m_info.samplers.clear(); + m_info.sampled_pairs.clear(); + } + + bool Run(std::string* error) { + if (m_program.resource_tracking_complete) { + return Fail(0, error, "resources already tracked"); + } + if (!m_program.srt_plan_complete) { + return Fail(0, error, "SRT plan is not ready"); + } + for (auto* block: m_values.blocks) { + for (auto& inst: *block) { + if (!Collect(inst, error)) { + return false; + } + } + } + LinkImageAliases(); + for (const auto& patch: m_handle_patches) { + patch.handle->SetFlags(patch.resource); + } + for (const auto& patch: m_memory_patches) { + auto& memory = m_values.memory_info[patch.index]; + memory.resource = patch.resource; + if (patch.has_sampler) { + memory.sampler = patch.sampler; + } + } + m_values.descriptor_sources = std::move(m_sources); + m_program.info = std::move(m_info); + m_program.resource_tracking_complete = true; + return true; + } + +private: + struct HandlePatch { + Inst* handle = nullptr; + uint32_t resource = 0; + }; + + struct MemoryPatch { + uint32_t index = 0; + uint32_t resource = 0; + uint32_t sampler = 0; + bool has_sampler = false; + }; + + bool Fail(uint32_t pc, std::string* error, const std::string& reason) const { + if (error != nullptr) { + *error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}", + m_program.shader_hash, StageName(m_program.stage), pc, reason); + } + return false; + } + + struct AddressPart { + Value value; + bool rooted = false; + }; + + AddressPart FindAddressPart(Value value, std::unordered_set& visiting) const { + value = value.Resolve(); + if (value.IsImmediate()) { + return {value, false}; + } + const auto* inst = value.TryInstruction(); + if (inst == nullptr || !visiting.insert(inst).second) { + return {}; + } + const auto finish = [&](AddressPart part) { + visiting.erase(inst); + return part; + }; + std::string reason; + if (ValidateRuntimeValue(m_program, value, reason)) { + return finish({value, true}); + } + const auto merge = [&](AddressPart left, AddressPart right, bool require_both) { + if (left.value.IsEmpty() || right.value.IsEmpty()) { + return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left); + } + return EquivalentValue(m_values, left.value, right.value) + ? AddressPart {left.value, left.rooted || right.rooted} + : AddressPart {}; + }; + switch (inst->GetOpcode()) { + case ValueOpcode::SelectU32: + return finish(merge(FindAddressPart(inst->Arg(1), visiting), + FindAddressPart(inst->Arg(2), visiting), true)); + case ValueOpcode::Phi: { + const auto invariant = ResolveInvariantPhi(m_values, value); + return finish(invariant.IsEmpty() ? AddressPart {} + : FindAddressPart(invariant, visiting)); + } + case ValueOpcode::IAdd32: + case ValueOpcode::ISub32: { + const auto left = FindAddressPart(inst->Arg(0), visiting); + const auto right = FindAddressPart(inst->Arg(1), visiting); + if (left.rooted == right.rooted || + (inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) { + return finish({}); + } + return finish(left.rooted ? left : right); + } + case ValueOpcode::CompositeExtractU32x2: { + const auto* source = inst->Arg(0).ResolveInstruction(); + if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) { + return finish({}); + } + const auto left = FindAddressPart(source->Arg(0), visiting); + const auto right = FindAddressPart(source->Arg(1), visiting); + if (left.rooted == right.rooted) { + return finish({}); + } + return finish(left.rooted ? left : right); + } + default: return finish({}); + } + } + + bool MakeFlatAddressSource(const Inst& handle, DescriptorSource& descriptor) const { + std::unordered_set visiting; + auto low = FindAddressPart(handle.Arg(0), visiting); + visiting.clear(); + auto high = FindAddressPart(handle.Arg(1), visiting); + if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) { + return false; + } + descriptor.dword_count = 2; + descriptor.dwords[0] = low.value; + descriptor.dwords[1] = high.value; + return true; + } + + bool MakeSource(const Inst& handle, uint32_t width, bool sampler, DescriptorSource& descriptor, + uint32_t pc, std::string* error) const { + if (handle.NumArgs() != width) { + return Fail(pc, error, + fmt::format("{} has {} descriptor dwords, expected {}", + ValueOpcodeName(handle.GetOpcode()), handle.NumArgs(), width)); + } + descriptor.dword_count = width; + for (uint32_t i = 0; i < width; i++) { + descriptor.dwords[i] = handle.Arg(i).Resolve(); + } + const auto dword0 = descriptor.dwords[0].Resolve(); + if (sampler && dword0.IsImmediate() && dword0.GetType() == Type::U32 && + (dword0.U32() & SamplerBorderClampMask) == 0) { + // Border color and its table index are unused unless a clamp axis selects border mode. + descriptor.dwords[3] = Value(0u); + } + return true; + } + + bool ValidateSource(const DescriptorSource& descriptor, std::string& reason, + uint32_t& bad_dword) const { + for (uint32_t i = 0; i < descriptor.dword_count; i++) { + bad_dword = i; + if (descriptor.dwords[i].Resolve().GetType() != Type::U32) { + reason = "is not U32"; + return false; + } + if (!ValidateRuntimeValue(m_program, descriptor.dwords[i], reason)) { + return false; + } + } + return true; + } + + uint32_t InternSource(const DescriptorSource& descriptor) { + for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) { + const auto& current = m_sources[candidate]; + if (current.dword_count != descriptor.dword_count) { + continue; + } + bool same = true; + for (uint32_t i = 0; i < descriptor.dword_count; i++) { + same = same && EquivalentValue(m_values, current.dwords[i], descriptor.dwords[i]); + } + if (same) { + return candidate; + } + } + m_sources.push_back(descriptor); + return static_cast(m_sources.size() - 1); + } + + bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle, + uint32_t& source, std::string* error, bool sampler = false) { + handle = value.Resolve().TryInstruction(); + if (handle == nullptr || handle->GetOpcode() != expected) { + return Fail(pc, error, + fmt::format("memory operation requires {}", ValueOpcodeName(expected))); + } + DescriptorSource descriptor; + if (!MakeSource(*handle, width, sampler, descriptor, pc, error)) { + return false; + } + std::string reason; + uint32_t bad_dword = 0; + if (!ValidateSource(descriptor, reason, bad_dword)) { + return Fail( + pc, error, + fmt::format("{} dword {} {}", ValueOpcodeName(expected), bad_dword, reason)); + } + source = InternSource(descriptor); + return true; + } + + bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle, + uint32_t& source, bool& unbased, std::string* error) { + handle = value.Resolve().TryInstruction(); + if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) { + return Fail(pc, error, "address operation requires GetAddressResource"); + } + if (handle->NumArgs() != 2) { + return Fail(pc, error, "GetAddressResource must have two address dwords"); + } + DescriptorSource descriptor; + const auto& memory = m_values.memory_info[memory_inst.Flags().index]; + if (memory.address_is_full) { + if (memory.kind != ResourceKind::Flat || !MakeFlatAddressSource(*handle, descriptor)) { + unbased = true; + source = UINT32_MAX; + return true; + } + } else if (!MakeSource(*handle, 2, false, descriptor, pc, error)) { + return false; + } + std::string reason; + uint32_t bad_dword = 0; + if (!ValidateSource(descriptor, reason, bad_dword)) { + if (memory.address_is_full) { + unbased = true; + source = UINT32_MAX; + return true; + } + return Fail( + pc, error, + fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason)); + } + unbased = false; + source = InternSource(descriptor); + return true; + } + + uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) { + for (uint32_t i = 0; i < m_info.buffers.size(); i++) { + if (m_info.buffers[i].source == source) { + Merge(m_info.buffers[i], memory, op, pc); + return i; + } + } + if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) { + return UINT32_MAX; + } + BufferResource resource; + resource.source = source; + resource.first_use_pc = pc; + Merge(resource, memory, op, pc); + m_info.buffers.push_back(resource); + return static_cast(m_info.buffers.size() - 1); + } + + static void Merge(BufferResource& resource, const MemoryInfo& memory, ValueOpcode op, + uint32_t pc) { + const bool atomic = IsBufferAtomic(op); + const bool write = IsBufferStore(op); + resource.first_use_pc = std::min(resource.first_use_pc, pc); + resource.max_byte_extent = std::max(resource.max_byte_extent, ByteExtent(memory)); + resource.read = resource.read || !write || atomic; + resource.written = resource.written || write; + resource.atomic = resource.atomic || atomic; + resource.formatted = resource.formatted || memory.formatted; + resource.scalar = resource.scalar || op == ValueOpcode::ReadConstBuffer || + memory.kind == ResourceKind::ScalarBuffer; + } + + uint32_t AddImage(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) { + const auto mip = MipMode(memory); + const bool depth = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0; + for (uint32_t i = 0; i < m_info.images.size(); i++) { + auto& image = m_info.images[i]; + if (image.source == source && image.kind == memory.kind && + image.dimension == memory.image_dimension && image.mip_mode == mip && + image.depth_compare == depth) { + Merge(image, op, pc); + return i; + } + } + if (m_info.images.size() >= ShaderInfo::MaxImages) { + return UINT32_MAX; + } + ImageResource image; + image.source = source; + image.first_use_pc = pc; + image.kind = memory.kind; + image.dimension = memory.image_dimension; + image.mip_mode = mip; + image.depth_compare = depth; + Merge(image, op, pc); + m_info.images.push_back(image); + return static_cast(m_info.images.size() - 1); + } + + static void Merge(ImageResource& image, ValueOpcode op, uint32_t pc) { + const bool atomic = IsImageAtomic(op); + const bool write = IsImageStore(op); + image.first_use_pc = std::min(image.first_use_pc, pc); + image.read = image.read || !write || atomic; + image.written = image.written || write; + image.atomic = image.atomic || atomic; + } + + uint32_t AddSampler(uint32_t source, uint32_t pc) { + for (uint32_t i = 0; i < m_info.samplers.size(); i++) { + if (m_info.samplers[i].source == source) { + m_info.samplers[i].first_use_pc = std::min(m_info.samplers[i].first_use_pc, pc); + return i; + } + } + if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) { + return UINT32_MAX; + } + m_info.samplers.push_back({source, pc}); + return static_cast(m_info.samplers.size() - 1); + } + + uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory, ValueOpcode op, + uint32_t pc) { + auto immediate = static_cast(memory.offset); + if (memory.kind == ResourceKind::ScalarBuffer) { + immediate = static_cast(static_cast(immediate) & ~3u); + } + const auto min_offset = unbased ? 0 : std::min(immediate, 0); + for (uint32_t i = 0; i < m_info.addresses.size(); i++) { + auto& address = m_info.addresses[i]; + if (address.source == source && address.unbased == unbased && + address.kind == memory.kind) { + address.first_use_pc = std::min(address.first_use_pc, pc); + address.min_offset = std::min(address.min_offset, min_offset); + address.read = address.read || !IsAddressStore(op); + address.written = address.written || IsAddressStore(op); + return i; + } + } + if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) { + return UINT32_MAX; + } + AddressResource address; + address.source = source; + address.first_use_pc = pc; + address.kind = memory.kind; + address.min_offset = min_offset; + address.unbased = unbased; + address.read = !IsAddressStore(op); + address.written = IsAddressStore(op); + m_info.addresses.push_back(address); + return static_cast(m_info.addresses.size() - 1); + } + + bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) { + for (auto& pair: m_info.sampled_pairs) { + if (pair.image == image && pair.sampler == sampler) { + pair.first_use_pc = std::min(pair.first_use_pc, pc); + return true; + } + } + if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) { + return Fail(pc, error, "sampled image/sampler pair limit exceeded"); + } + m_info.sampled_pairs.push_back({image, sampler, pc}); + return true; + } + + bool AddHandlePatch(Inst* handle, uint32_t resource, uint32_t pc, std::string* error) { + for (const auto& patch: m_handle_patches) { + if (patch.handle == handle) { + return patch.resource == resource || + Fail(pc, error, + fmt::format("{} is reused with incompatible resource classes", + ValueOpcodeName(handle->GetOpcode()))); + } + } + m_handle_patches.push_back({handle, resource}); + return true; + } + + bool AddMemoryPatch(uint32_t index, uint32_t resource, uint32_t sampler, bool has_sampler, + uint32_t pc, std::string* error) { + for (auto& patch: m_memory_patches) { + if (patch.index != index) { + continue; + } + if (patch.resource != resource || + (has_sampler && patch.has_sampler && patch.sampler != sampler)) { + return Fail(pc, error, "memory metadata is reused with incompatible resources"); + } + if (has_sampler) { + patch.sampler = sampler; + patch.has_sampler = true; + } + return true; + } + m_memory_patches.push_back({index, resource, sampler, has_sampler}); + return true; + } + + bool Collect(Inst& inst, std::string* error) { + const auto op = inst.GetOpcode(); + if (!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) { + return true; + } + const auto flags = inst.Flags(); + if (flags.index >= m_values.memory_info.size()) { + return Fail(flags.pc, error, + fmt::format("memory metadata index {} is out of range", flags.index)); + } + if (inst.NumArgs() == 0) { + return Fail(flags.pc, error, "memory operation has no resource handle"); + } + const auto& memory = m_values.memory_info[flags.index]; + if (memory.planning_only) { + return true; + } + Inst* handle = nullptr; + uint32_t source = 0; + uint32_t resource = 0; + + if (IsBuffer(op)) { + if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source, + error)) { + return false; + } + resource = AddBuffer(source, memory, op, flags.pc); + if (resource == UINT32_MAX) { + return Fail(flags.pc, error, "buffer resource limit exceeded"); + } + return AddHandlePatch(handle, resource, flags.pc, error) && + AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error); + } + if (IsAddress(op)) { + bool unbased = false; + if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) { + return false; + } + resource = AddAddress(source, unbased, memory, op, flags.pc); + if (resource == UINT32_MAX) { + return Fail(flags.pc, error, "address resource limit exceeded"); + } + return AddHandlePatch(handle, resource, flags.pc, error) && + AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error); + } + + if (!GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, source, + error)) { + return false; + } + resource = AddImage(source, memory, op, flags.pc); + if (resource == UINT32_MAX) { + return Fail(flags.pc, error, "image resource limit exceeded"); + } + if (!AddHandlePatch(handle, resource, flags.pc, error)) { + return false; + } + uint32_t sampler = 0; + if (NeedsSampler(op)) { + if (inst.NumArgs() < 2) { + return Fail(flags.pc, error, "sampled image operation has no sampler handle"); + } + Inst* sampler_handle = nullptr; + uint32_t sampler_source = 0; + if (!GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc, + sampler_handle, sampler_source, error, true)) { + return false; + } + sampler = AddSampler(sampler_source, flags.pc); + if (sampler == UINT32_MAX) { + return Fail(flags.pc, error, "sampler resource limit exceeded"); + } + if (!AddHandlePatch(sampler_handle, sampler, flags.pc, error) || + !AddSampledPair(resource, sampler, flags.pc, error)) { + return false; + } + } + return AddMemoryPatch(flags.index, resource, sampler, NeedsSampler(op), flags.pc, error); + } + + const DescriptorSource* Source(uint32_t source) const { + return source < m_sources.size() ? &m_sources[source] : nullptr; + } + + void LinkImageAliases() { + for (auto& buffer: m_info.buffers) { + const auto* buffer_source = Source(buffer.source); + if (buffer_source == nullptr || buffer_source->dword_count != 4) { + continue; + } + for (uint32_t image = 0; image < m_info.images.size(); image++) { + const auto* image_source = Source(m_info.images[image].source); + if (image_source == nullptr || image_source->dword_count != 8) { + continue; + } + bool alias = true; + for (uint32_t dword = 0; dword < 4; dword++) { + alias = alias && EquivalentValue(m_values, buffer_source->dwords[dword], + image_source->dwords[dword]); + } + if (alias) { + buffer.image_alias = image; + break; + } + } + } + } + + Program& m_program; + ValueProgram& m_values; + ShaderInfo m_info; + std::vector m_sources; + std::vector m_handle_patches; + std::vector m_memory_patches; +}; + +} // namespace + +bool TrackResources(Program& program, std::string* error) { + if (program.values == nullptr) { + if (error != nullptr) { + *error = "shader resource tracking requires typed IR"; + } + return false; + } + return Tracker(program, *program.values).Run(error); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ResourceTracking.h b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h similarity index 72% rename from src/graphics/shader/recompiler/ir/ResourceTracking.h rename to src/graphics/shader/recompiler/ir/passes/ResourceTracking.h index f1fc4db77..791424879 100644 --- a/src/graphics/shader/recompiler/ir/ResourceTracking.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.h @@ -5,8 +5,8 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -// Collects immutable resource topology, then replaces descriptor operands with dense indices. -// On failure the program is left unpatched. +// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in +// descriptor_sources, then writes dense indices to handle flags and MemoryInfo. bool TrackResources(Program& program, std::string* error); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp new file mode 100644 index 000000000..51f7aaa20 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp @@ -0,0 +1,318 @@ +#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" + +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components, + std::string name) { + const auto input = std::find_if(info.inputs.begin(), info.inputs.end(), [=](const auto& value) { + return value.kind == kind && value.location == location; + }); + if (input == info.inputs.end()) { + info.inputs.push_back({kind, location, components, std::move(name)}); + } else { + input->component_count = std::max(input->component_count, components); + } +} + +bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) { + return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) { + return output.kind == kind && output.index == index; + }); +} + +void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t location, + std::string name) { + if (!HasOutput(info, kind, index)) { + info.outputs.push_back({kind, index, location, std::move(name)}); + } +} + +bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) { + auto Fail = [&](const char* message) { + if (error != nullptr) { + *error = message; + } + return false; + }; + switch (program.stage) { + case ShaderType::Vertex: + if (options.vertex == nullptr) { + return Fail("vertex shader info requires vertex metadata"); + } + if (options.vertex->resources_num < 0 || + options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) { + return Fail("vertex resource count is out of range"); + } + return true; + case ShaderType::Pixel: + if (options.pixel == nullptr) { + return Fail("pixel shader info requires pixel metadata"); + } + return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) || + Fail("pixel input count is out of range"); + case ShaderType::Compute: + if (options.compute == nullptr) { + return Fail("compute shader info requires compute metadata"); + } + return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) || + Fail("compute thread ID count is out of range"); + default: return Fail("unsupported shader stage for info collection"); + } +} + +bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& options, + std::string* error) { + const auto Fail = [&](const char* message) { + if (error != nullptr) { + *error = message; + } + return false; + }; + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + switch (inst.GetOpcode()) { + case ValueOpcode::GetAttribute: { + if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 || + !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) { + return Fail("typed attribute reference is not constant"); + } + if (program.stage == ShaderType::Vertex && + (inst.Arg(1).U32() >= 4u || options.vertex == nullptr || + inst.Arg(0).U32() >= + static_cast(options.vertex->resources_num))) { + return Fail("vertex input reference is out of range"); + } + break; + } + case ValueOpcode::GetBuiltin: { + if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 || + !inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) { + return Fail("typed builtin reference is not constant"); + } + const auto kind = static_cast(inst.Arg(0).U32()); + const auto component = inst.Arg(1).U32(); + switch (kind) { + case StageInputKind::VertexIndex: + case StageInputKind::InstanceIndex: + case StageInputKind::FrontFacing: + case StageInputKind::LocalInvocationIndex: + if (component != 0u) { + return Fail("typed scalar builtin component is out of range"); + } + break; + case StageInputKind::FragCoord: + if (component >= 4u) { + return Fail("typed fragment-coordinate component is out of range"); + } + break; + case StageInputKind::WorkgroupId: + case StageInputKind::LocalInvocationId: + case StageInputKind::GlobalInvocationId: + if (component >= 3u) { + return Fail("typed invocation builtin component is out of range"); + } + break; + case StageInputKind::Parameter: + default: return Fail("typed builtin kind is invalid"); + } + break; + } + case ValueOpcode::SetAttribute: + if (inst.Flags().index >= program.values->export_info.size()) { + return Fail("typed export metadata index is out of range"); + } + break; + default: break; + } + } + } + return true; +} + +void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex, + ShaderInfo& info) { + AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex"); + AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex"); + if (vertex == nullptr) { + return; + } + uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {}; + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() == ValueOpcode::GetAttribute) { + const auto attr = inst.Arg(0).U32(); + const auto chan = inst.Arg(1).U32(); + used_components[attr] = std::max(used_components[attr], chan + 1u); + } + } + } + for (uint32_t attr = 0; attr < static_cast(vertex->resources_num) && + attr < ShaderVertexInputInfo::RES_MAX; + attr++) { + if (used_components[attr] != 0) { + AddInput(info, StageInputKind::Parameter, attr, used_components[attr], + fmt::format("in_attr_{}", attr)); + } + } +} + +void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) { + if (pixel == nullptr) { + return; + } + if (pixel->HasPositionInput()) { + AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord"); + } + if (pixel->ps_front_face) { + AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing"); + } + for (uint32_t input = 0; input < pixel->input_num; input++) { + AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input)); + } +} + +void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& info) { + if (compute != nullptr) { + if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) { + AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID"); + } + if (compute->thread_ids_num > 0) { + AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID"); + } + if (compute->thread_ids_num > 0 || compute->tg_size_en) { + AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); + } + if (compute->dispatch_thread_dimensions) { + AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID"); + } + } +} + +void CollectBuiltinInputs(const Program& program, ShaderInfo& info) { + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() != ValueOpcode::GetBuiltin) { + continue; + } + const auto kind = static_cast(inst.Arg(0).U32()); + switch (kind) { + case StageInputKind::VertexIndex: + AddInput(info, kind, 0, 1, "gl_VertexIndex"); + break; + case StageInputKind::InstanceIndex: + AddInput(info, kind, 0, 1, "gl_InstanceIndex"); + break; + case StageInputKind::FragCoord: AddInput(info, kind, 0, 4, "gl_FragCoord"); break; + case StageInputKind::FrontFacing: + AddInput(info, kind, 0, 1, "gl_FrontFacing"); + break; + case StageInputKind::WorkgroupId: + AddInput(info, kind, 0, 3, "gl_WorkGroupID"); + break; + case StageInputKind::LocalInvocationId: + AddInput(info, kind, 0, 3, "gl_LocalInvocationID"); + break; + case StageInputKind::LocalInvocationIndex: + AddInput(info, kind, 0, 1, "gl_LocalInvocationIndex"); + break; + case StageInputKind::GlobalInvocationId: + AddInput(info, kind, 0, 3, "gl_GlobalInvocationID"); + break; + case StageInputKind::Parameter: break; + } + } + } +} + +void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo& info) { + for (const auto* block: program.values->blocks) { + for (const auto& inst: *block) { + if (inst.GetOpcode() != ValueOpcode::SetAttribute) { + continue; + } + const auto& export_info = program.values->export_info[inst.Flags().index]; + if (export_info.kind == ExportTargetKind::MrtZ) { + if (pixel != nullptr && (export_info.en & 0x1u) != 0 && + pixel->ps_depth_export_enable) { + AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth"); + } + if (pixel != nullptr && (export_info.en & 0x4u) != 0 && + pixel->ps_sample_mask_export_enable) { + AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask"); + } + continue; + } + if (export_info.en == 0) { + continue; + } + switch (export_info.kind) { + case ExportTargetKind::Position: + AddOutput(info, StageOutputKind::Position, export_info.index, 0, + "out_position"); + break; + case ExportTargetKind::Parameter: + AddOutput(info, StageOutputKind::Parameter, export_info.index, + export_info.index, fmt::format("out_param_{}", export_info.index)); + break; + case ExportTargetKind::Mrt: + AddOutput(info, StageOutputKind::Mrt, export_info.index, export_info.index, + fmt::format("out_mrt_{}", export_info.index)); + break; + default: break; + } + } + } +} + +} // namespace + +bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) { + if (!program.resource_tracking_complete || program.shader_info_complete) { + if (error != nullptr) { + *error = !program.resource_tracking_complete ? "shader resources were not tracked" + : "shader info already collected"; + } + return false; + } + if (program.values == nullptr) { + if (error != nullptr) { + *error = "typed value program is not available"; + } + return false; + } + if (!ValidateOptions(program, options, error)) { + return false; + } + if (!ValidateValueReferences(program, options, error)) { + return false; + } + + auto next = program.info; + next.inputs.clear(); + next.outputs.clear(); + next.has_bitwise_xor = std::any_of( + program.values->blocks.begin(), program.values->blocks.end(), [](const auto* block) { + return std::any_of(block->begin(), block->end(), [](const auto& inst) { + return inst.GetOpcode() == ValueOpcode::BitwiseXor32; + }); + }); + switch (program.stage) { + case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break; + case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break; + case ShaderType::Compute: CollectComputeInputs(options.compute, next); break; + default: return false; + } + CollectBuiltinInputs(program, next); + CollectOutputs(program, options.pixel, next); + program.info = std::move(next); + program.shader_info_complete = true; + return true; +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/ShaderInfoCollection.h b/src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h similarity index 100% rename from src/graphics/shader/recompiler/ir/ShaderInfoCollection.h rename to src/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp new file mode 100644 index 000000000..51d5eb10f --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -0,0 +1,951 @@ +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" + +#include "graphics/shader/recompiler/ir/ValueProgram.h" + +#include +#include +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +constexpr uint64_t AddressMask = 0x0000ffffffffffffull; + +const char* StageName(ShaderType stage) { + switch (stage) { + case ShaderType::Vertex: return "vertex"; + case ShaderType::Pixel: return "pixel"; + case ShaderType::Fetch: return "fetch"; + case ShaderType::Compute: return "compute"; + default: return "unknown"; + } +} + +std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) { + return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash, + StageName(program.stage), pc, message); +} + +bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { + if (base > AddressMask) { + return false; + } + if (offset < 0) { + const auto magnitude = uint64_t {0} - static_cast(offset); + if (magnitude > base) { + return false; + } + result = base - magnitude; + return true; + } + const auto magnitude = static_cast(offset); + if (magnitude > AddressMask - base) { + return false; + } + result = base + magnitude; + return true; +} + +bool IsRawRead(const ValueProgram& values, const Inst& inst) { + if (inst.GetOpcode() != ValueOpcode::LoadAddressU32 && + inst.GetOpcode() != ValueOpcode::ReadConstBuffer) { + return false; + } + const auto index = inst.Flags().index; + return index < values.memory_info.size() && + values.memory_info[index].kind == ResourceKind::ScalarBuffer; +} + +bool IsDescriptorHandle(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::GetBufferResource: + case ValueOpcode::GetAddressResource: + case ValueOpcode::GetImageResource: + case ValueOpcode::GetSamplerResource: return true; + default: return false; + } +} + +bool IsRuntimeIntegerOp(ValueOpcode op) { + switch (op) { + case ValueOpcode::CompositeConstructU32x2: + case ValueOpcode::CompositeExtractU32x2: + case ValueOpcode::PackUint2x32: + case ValueOpcode::UnpackUint2x32: + case ValueOpcode::BitFieldInsert: + case ValueOpcode::BitFieldUExtract: + case ValueOpcode::BitFieldSExtract: + case ValueOpcode::IAdd32: + case ValueOpcode::IAdd64: + case ValueOpcode::IAddCarry32: + case ValueOpcode::ISub32: + case ValueOpcode::ISub64: + case ValueOpcode::IMul32: + case ValueOpcode::IMul64: + case ValueOpcode::ShiftLeftLogical32: + case ValueOpcode::ShiftLeftLogical64: + case ValueOpcode::ShiftRightLogical32: + case ValueOpcode::ShiftRightLogical64: + case ValueOpcode::ShiftRightArithmetic32: + case ValueOpcode::ShiftRightArithmetic64: + case ValueOpcode::BitwiseAnd32: + case ValueOpcode::BitwiseAnd64: + case ValueOpcode::BitwiseOr32: + case ValueOpcode::BitwiseOr64: + case ValueOpcode::BitwiseXor32: + case ValueOpcode::BitwiseNot32: + case ValueOpcode::SelectU1: + case ValueOpcode::SelectU32: + case ValueOpcode::ULessThan32: + case ValueOpcode::IEqual32: + case ValueOpcode::UGreaterThan32: + case ValueOpcode::INotEqual32: + case ValueOpcode::LogicalOr: + case ValueOpcode::LogicalAnd: + case ValueOpcode::LogicalXor: + case ValueOpcode::LogicalNot: return true; + default: return false; + } +} + +class RuntimeValidator { +public: + RuntimeValidator(const Program& program, const ValueProgram& values) + : m_program(program), m_values(values) {} + + bool Run(Value value, std::string& reason) { return Validate(value, reason); } + +private: + bool Validate(Value value, std::string& reason) { + value = value.Resolve(); + const auto* inst = value.TryInstruction(); + if (inst == nullptr) { + switch (value.GetType()) { + case Type::U1: + case Type::U8: + case Type::U16: + case Type::U32: + case Type::U64: return true; + default: + reason = fmt::format("contains a non-integer immediate of type {}", + TypeName(value.GetType())); + return false; + } + } + if (!m_visiting.insert(inst).second) { + reason = fmt::format("contains a cyclic {} value", ValueOpcodeName(inst->GetOpcode())); + return false; + } + const auto finish = [&](bool valid) { + m_visiting.erase(inst); + return valid; + }; + const auto op = inst->GetOpcode(); + if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU8 || + op == ValueOpcode::UndefU16 || op == ValueOpcode::UndefU32 || + op == ValueOpcode::UndefU64 || op == ValueOpcode::Void) { + reason = fmt::format("contains {}", ValueOpcodeName(op)); + return finish(false); + } + if (op == ValueOpcode::GetUserData) { + if (inst->NumArgs() != 1 || inst->Arg(0).GetType() != Type::ScalarReg) { + reason = "contains a malformed GetUserData"; + return finish(false); + } + const auto reg = RegIndex(inst->Arg(0).ScalarRegister()); + if (reg < m_program.user_data_base || + reg - m_program.user_data_base >= m_program.user_data_count) { + reason = fmt::format("references unavailable user SGPR {}", reg); + return finish(false); + } + return finish(true); + } + if (op == ValueOpcode::GetShaderBase) { + if (inst->NumArgs() != 0) { + reason = "contains a malformed GetShaderBase"; + return finish(false); + } + return finish(true); + } + if (op == ValueOpcode::Phi) { + const auto invariant = ResolveInvariantPhi(m_values, value); + if (invariant.IsEmpty()) { + reason = "contains a control-dependent phi"; + return finish(false); + } + return finish(Validate(invariant, reason)); + } + if (op == ValueOpcode::GetSrtResource) { + if (inst->NumArgs() != 0) { + reason = "contains a malformed GetSrtResource"; + return finish(false); + } + return finish(true); + } + if (op == ValueOpcode::ReadConst) { + const auto slot = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {}; + if (inst->NumArgs() != 2 || inst->Arg(0).Resolve().TryInstruction() == nullptr || + inst->Arg(0).Resolve().TryInstruction()->GetOpcode() != + ValueOpcode::GetSrtResource || + !slot.IsImmediate() || slot.GetType() != Type::U32 || + slot.U32() >= m_values.srt_reads.size()) { + reason = "contains a malformed flattened SRT read"; + return finish(false); + } + } else if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) { + const auto expected = op == ValueOpcode::LoadAddressU32 + ? ValueOpcode::GetAddressResource + : ValueOpcode::GetBufferResource; + const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr; + if (!IsRawRead(m_values, *inst) || handle == nullptr || + handle->GetOpcode() != expected) { + reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op)); + return finish(false); + } + } else if (op == ValueOpcode::PackUint2x32) { + const auto* pair = inst->NumArgs() == 1 ? inst->Arg(0).ResolveInstruction() : nullptr; + if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2) { + reason = "contains an unsupported packed runtime value"; + return finish(false); + } + } else if (op == ValueOpcode::CompositeExtractU32x2) { + const auto* source = inst->NumArgs() == 2 ? inst->Arg(0).ResolveInstruction() : nullptr; + const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {}; + if (source == nullptr || !index.IsImmediate() || index.GetType() != Type::U32 || + index.U32() >= 2u || + (source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 && + source->GetOpcode() != ValueOpcode::UnpackUint2x32 && + source->GetOpcode() != ValueOpcode::IAddCarry32)) { + reason = "contains an unsupported composite runtime source"; + return finish(false); + } + } + if (op == ValueOpcode::GetBufferResource || op == ValueOpcode::GetImageResource || + op == ValueOpcode::GetSamplerResource || op == ValueOpcode::GetAddressResource) { + const size_t expected = op == ValueOpcode::GetBufferResource ? 4u + : op == ValueOpcode::GetImageResource ? 8u + : op == ValueOpcode::GetSamplerResource ? 4u + : 2u; + if (inst->NumArgs() != expected) { + reason = fmt::format("contains a malformed {}", ValueOpcodeName(op)); + return finish(false); + } + } else if (op != ValueOpcode::ReadConst && op != ValueOpcode::ReadConstBuffer && + op != ValueOpcode::LoadAddressU32 && !IsRuntimeIntegerOp(op)) { + reason = + fmt::format("contains unsupported or control-dependent {}", ValueOpcodeName(op)); + return finish(false); + } + for (size_t index = 0; index < inst->NumArgs(); index++) { + if (!Validate(inst->Arg(index), reason)) { + return finish(false); + } + } + return finish(true); + } + + const Program& m_program; + const ValueProgram& m_values; + std::unordered_set m_visiting; +}; + +class PlanBuilder { +public: + PlanBuilder(Program& program, ValueProgram& values): m_program(program), m_values(values) {} + + bool Run(std::string* error) { + m_values.srt_reads.clear(); + m_values.dynamic_reads.clear(); + for (auto* block: m_values.blocks) { + for (auto& inst: *block) { + if (IsDescriptorHandle(inst.GetOpcode())) { + for (size_t index = 0; index < inst.NumArgs(); index++) { + if (!Collect(inst.Arg(index), 0, error)) { + return false; + } + } + } + } + } + for (auto* block: m_values.blocks) { + for (auto& inst: *block) { + std::string reason; + if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_values, inst) && + inst.Arg(1).Resolve().IsImmediate() && + ValidateRuntimeValue(m_program, Value(&inst), reason) && + !Collect(Value(&inst), inst.Flags().pc, error)) { + return false; + } + } + } + PatchReads(); + return true; + } + +private: + struct Patch { + Inst* inst = nullptr; + uint32_t slot = 0; + bool keep = false; + }; + + bool Fail(uint32_t pc, std::string* error, const std::string& message) const { + if (error != nullptr) { + *error = Diagnostic(m_program, pc, message); + } + return false; + } + + bool Collect(Value value, uint32_t use_pc, std::string* error) { + value = value.Resolve(); + if (value.IsImmediate()) { + return true; + } + auto* inst = value.TryInstruction(); + if (inst == nullptr) { + return Fail(use_pc, error, "invalid typed planning value"); + } + if (std::ranges::find(m_visiting, inst) != m_visiting.end()) { + return inst->GetOpcode() == ValueOpcode::Phi; + } + if (std::ranges::find(m_visited, inst) != m_visited.end()) { + return true; + } + m_visiting.push_back(inst); + for (size_t index = 0; index < inst->NumArgs(); index++) { + if (!Collect(inst->Arg(index), use_pc, error)) { + return false; + } + } + m_visiting.pop_back(); + m_visited.push_back(inst); + if (!IsRawRead(m_values, *inst)) { + return true; + } + const auto offset = inst->Arg(1).Resolve(); + if (!offset.IsImmediate() || offset.GetType() != Type::U32) { + if (std::ranges::find(m_values.dynamic_reads, value) == m_values.dynamic_reads.end()) { + m_values.dynamic_reads.push_back(value); + } + return true; + } + use_pc = inst->Flags().pc; + for (uint32_t slot = 0; slot < m_values.srt_reads.size(); slot++) { + if (EquivalentValue(m_values, value, m_values.srt_reads[slot].value)) { + m_patches.push_back({inst, slot, false}); + return true; + } + } + const auto slot = static_cast(m_values.srt_reads.size()); + m_values.srt_reads.push_back({value, slot, use_pc}); + m_patches.push_back({inst, slot, true}); + return true; + } + + void PatchReads() { + for (const auto& patch: m_patches) { + auto* block = patch.inst->Parent(); + auto& list = block->Instructions(); + auto where = + std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; }); + const auto resource = + Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource)); + const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst, + {resource, Value(patch.slot)})); + const auto uses = patch.inst->Uses(); + for (const auto& use: uses) { + use.user->SetArg(use.operand, flat); + } + for (auto& info: m_values.block_info) { + if (info.condition.Resolve() == Value(patch.inst)) { + info.condition = flat; + } + if (info.indirect_target.Resolve() == Value(patch.inst)) { + info.indirect_target = flat; + } + } + if (patch.keep) { + const auto memory = patch.inst->Flags().index; + if (memory < m_values.memory_info.size()) { + m_values.memory_info[memory].planning_only = true; + } + block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)}); + } + } + } + + Program& m_program; + ValueProgram& m_values; + std::vector m_visiting; + std::vector m_visited; + std::vector m_patches; +}; + +class Evaluator { +public: + Evaluator(const Program& program, const SrtRuntime& runtime) + : m_program(program), m_values(*program.values), m_runtime(runtime) {} + + void SetUsePc(uint32_t pc) { m_use_pc = pc; } + + bool Evaluate(Value value, uint32_t& result, std::string* error) { + uint64_t wide = 0; + if (!EvaluateWide(value, wide, error)) { + return false; + } + result = static_cast(wide); + return true; + } + +private: + bool Fail(std::string* error, const std::string& message) const { + if (error != nullptr) { + *error = Diagnostic(m_program, m_use_pc, message); + } + return false; + } + + bool EvaluateWide(Value value, uint64_t& result, std::string* error) { + value = value.Resolve(); + if (value.IsImmediate()) { + switch (value.GetType()) { + case Type::U1: result = value.U1(); return true; + case Type::U8: result = value.U8(); return true; + case Type::U16: result = value.U16(); return true; + case Type::U32: result = value.U32(); return true; + case Type::U64: result = value.U64(); return true; + default: return Fail(error, "non-integer immediate in runtime expression"); + } + } + auto* inst = value.TryInstruction(); + if (inst == nullptr) { + return Fail(error, "invalid typed runtime value"); + } + if (const auto found = m_cache.find(inst); found != m_cache.end()) { + result = found->second; + return true; + } + if (std::ranges::find(m_visiting, inst) != m_visiting.end()) { + return Fail(error, "cyclic typed runtime value"); + } + m_visiting.push_back(inst); + uint64_t out = 0; + if (!EvaluateInst(*inst, out, error)) { + return false; + } + m_visiting.pop_back(); + m_cache.emplace(inst, out); + result = out; + return true; + } + + bool Arg(const Inst& inst, size_t index, uint64_t& result, std::string* error) { + return EvaluateWide(inst.Arg(index), result, error); + } + + bool EvaluatePhi(const Inst& inst, uint64_t& result, std::string* error) { + const auto value = ResolveInvariantPhi(m_values, Value(const_cast(&inst))); + return !value.IsEmpty() ? EvaluateWide(value, result, error) + : Fail(error, "typed phi has runtime-dependent values"); + } + + bool EvaluateExtract(const Inst& inst, uint64_t& result, std::string* error) { + const auto index = inst.Arg(1).Resolve(); + if (!index.IsImmediate() || index.GetType() != Type::U32) { + return Fail(error, "dynamic composite extract in runtime expression"); + } + const auto component = index.U32(); + const auto* source = inst.Arg(0).ResolveInstruction(); + if (source == nullptr || component >= 2u) { + return Fail(error, "unsupported composite runtime source"); + } + if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) { + return EvaluateWide(source->Arg(component), result, error); + } + if (source->GetOpcode() == ValueOpcode::UnpackUint2x32) { + uint64_t packed = 0; + if (!Arg(*source, 0, packed, error)) { + return false; + } + result = static_cast(packed >> (component * 32u)); + return true; + } + if (source->GetOpcode() == ValueOpcode::IAddCarry32) { + uint64_t lhs = 0; + uint64_t rhs = 0; + if (!Arg(*source, 0, lhs, error) || !Arg(*source, 1, rhs, error)) { + return false; + } + const auto sum = + static_cast(static_cast(lhs)) + static_cast(rhs); + result = + component == 0u ? static_cast(sum) : static_cast(sum >> 32u); + return true; + } + return Fail(error, "unsupported composite runtime source"); + } + + bool EvaluateRawRead(const Inst& inst, uint64_t& result, std::string* error) { + const auto flags = inst.Flags(); + if (flags.index >= m_values.memory_info.size()) { + return Fail(error, "raw scalar read has invalid metadata"); + } + const auto& mem = m_values.memory_info[flags.index]; + const auto* handle = inst.Arg(0).ResolveInstruction(); + if (handle == nullptr) { + return Fail(error, "raw scalar read has no descriptor handle"); + } + uint64_t low = 0; + uint64_t high = 0; + uint64_t offset = 0; + if (!Arg(*handle, 0, low, error) || !Arg(*handle, 1, high, error) || + !Arg(inst, 1, offset, error)) { + return false; + } + const auto base = ((high << 32u) | static_cast(low)) & AddressMask; + const auto immediate = static_cast(static_cast(mem.offset)); + uint64_t address = 0; + if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) { + uint64_t records = 0; + uint64_t word3 = 0; + if (handle->NumArgs() != 4u || !Arg(*handle, 2, records, error) || + !Arg(*handle, 3, word3, error)) { + return Fail(error, "constant-buffer descriptor has invalid width"); + } + if (immediate < 0) { + return Fail(error, "constant-buffer read has a negative immediate offset"); + } + const auto byte_offset = + static_cast(immediate) + static_cast(offset); + const auto aligned = byte_offset & ~uint64_t {3}; + const auto stride = (static_cast(high) >> 16u) & 0x3fffu; + const auto size = stride == 0u + ? static_cast(static_cast(records)) + : static_cast(stride) * static_cast(records); + if (aligned > size || size - aligned < sizeof(uint32_t)) { + return Fail( + error, fmt::format("constant-buffer offset {} exceeds size {}", aligned, size)); + } + address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3}; + } else { + const auto relative = (immediate & ~int64_t {3}) + + static_cast(static_cast(offset) & ~3u); + if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) { + return Fail(error, "raw scalar read is outside the 48-bit address space"); + } + } + uint32_t word = 0; + if (m_runtime.read_memory != nullptr) { + if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) { + return Fail(error, fmt::format("constant read failed at 0x{:016x}", address)); + } + } else { + std::memcpy(&word, reinterpret_cast(address), sizeof(word)); + } + result = word; + return true; + } + + bool EvaluateInst(const Inst& inst, uint64_t& result, std::string* error) { + uint64_t a = 0; + uint64_t b = 0; + uint64_t c = 0; + const auto binary = [&]() { return Arg(inst, 0, a, error) && Arg(inst, 1, b, error); }; + const auto ternary = [&]() { + return Arg(inst, 0, a, error) && Arg(inst, 1, b, error) && Arg(inst, 2, c, error); + }; + switch (inst.GetOpcode()) { + case ValueOpcode::GetUserData: { + const auto reg = RegIndex(inst.Arg(0).ScalarRegister()); + if (reg < m_program.user_data_base || + reg - m_program.user_data_base >= m_runtime.user_data.size()) { + return Fail(error, fmt::format("user SGPR {} is unavailable", reg)); + } + result = m_runtime.user_data[reg - m_program.user_data_base]; + return true; + } + case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true; + case ValueOpcode::Phi: return EvaluatePhi(inst, result, error); + case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result, error); + case ValueOpcode::PackUint2x32: { + const auto* pair = inst.Arg(0).ResolveInstruction(); + if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2 || + !Arg(*pair, 0, a, error) || !Arg(*pair, 1, b, error)) { + return Fail(error, "unsupported packed runtime value"); + } + result = static_cast(a) | + (static_cast(static_cast(b)) << 32u); + return true; + } + case ValueOpcode::ReadConst: { + const auto slot = inst.Arg(1).Resolve(); + if (!slot.IsImmediate() || slot.GetType() != Type::U32 || + slot.U32() >= m_values.srt_reads.size()) { + return Fail(error, "invalid flattened SRT index"); + } + return EvaluateWide(m_values.srt_reads[slot.U32()].value, result, error); + } + case ValueOpcode::LoadAddressU32: + case ValueOpcode::ReadConstBuffer: + if (IsRawRead(m_values, inst)) { + return EvaluateRawRead(inst, result, error); + } + break; + case ValueOpcode::IAdd32: + if (binary()) { + result = static_cast(a + b); + return true; + } + return false; + case ValueOpcode::IAdd64: + if (binary()) { + result = a + b; + return true; + } + return false; + case ValueOpcode::ISub32: + if (binary()) { + result = static_cast(a - b); + return true; + } + return false; + case ValueOpcode::ISub64: + if (binary()) { + result = a - b; + return true; + } + return false; + case ValueOpcode::IMul32: + if (binary()) { + result = static_cast(a * b); + return true; + } + return false; + case ValueOpcode::IMul64: + if (binary()) { + result = a * b; + return true; + } + return false; + case ValueOpcode::BitwiseAnd32: + if (binary()) { + result = static_cast(a & b); + return true; + } + return false; + case ValueOpcode::BitwiseAnd64: + if (binary()) { + result = a & b; + return true; + } + return false; + case ValueOpcode::BitwiseOr32: + if (binary()) { + result = static_cast(a | b); + return true; + } + return false; + case ValueOpcode::BitwiseOr64: + if (binary()) { + result = a | b; + return true; + } + return false; + case ValueOpcode::BitwiseXor32: + if (binary()) { + result = static_cast(a ^ b); + return true; + } + return false; + case ValueOpcode::BitwiseNot32: + if (Arg(inst, 0, a, error)) { + result = ~static_cast(a); + return true; + } + return false; + case ValueOpcode::ShiftLeftLogical32: + if (binary()) { + result = static_cast(a) << (b & 31u); + return true; + } + return false; + case ValueOpcode::ShiftLeftLogical64: + if (binary()) { + result = a << (b & 63u); + return true; + } + return false; + case ValueOpcode::ShiftRightLogical32: + if (binary()) { + result = static_cast(a) >> (b & 31u); + return true; + } + return false; + case ValueOpcode::ShiftRightLogical64: + if (binary()) { + result = a >> (b & 63u); + return true; + } + return false; + case ValueOpcode::ShiftRightArithmetic32: + if (binary()) { + result = static_cast( + std::bit_cast(static_cast(a)) >> (b & 31u)); + return true; + } + return false; + case ValueOpcode::ShiftRightArithmetic64: + if (binary()) { + result = static_cast(std::bit_cast(a) >> (b & 63u)); + return true; + } + return false; + case ValueOpcode::BitFieldUExtract: + if (ternary()) { + const auto offset = static_cast(b); + const auto width = static_cast(c); + if (offset > 32u || width > 32u - offset) { + return Fail(error, "invalid unsigned bit-field range"); + } + const auto mask = width == 32u ? UINT32_MAX + : width == 0u ? 0u + : (uint32_t {1} << width) - 1u; + result = width == 0u ? 0u : (static_cast(a) >> offset) & mask; + return true; + } + return false; + case ValueOpcode::BitFieldSExtract: + if (ternary()) { + const auto offset = static_cast(b); + const auto width = static_cast(c); + if (offset > 32u || width > 32u - offset) { + return Fail(error, "invalid signed bit-field range"); + } + if (width == 0u) { + result = 0; + return true; + } + const auto mask = width == 32u ? UINT32_MAX : (uint32_t {1} << width) - 1u; + auto bits = (static_cast(a) >> offset) & mask; + if (width < 32u && (bits & (uint32_t {1} << (width - 1u))) != 0u) { + bits |= ~mask; + } + result = bits; + return true; + } + return false; + case ValueOpcode::BitFieldInsert: { + uint64_t d = 0; + if (!ternary() || !Arg(inst, 3, d, error)) { + return false; + } + const auto offset = static_cast(c); + const auto width = static_cast(d); + if (offset > 32u || width > 32u - offset) { + return Fail(error, "invalid inserted bit-field range"); + } + if (width == 0u) { + result = static_cast(a); + return true; + } + const auto mask = + width == 32u ? UINT32_MAX : ((uint32_t {1} << width) - 1u) << offset; + result = (static_cast(a) & ~mask) | + ((static_cast(b) << offset) & mask); + return true; + } + case ValueOpcode::SelectU32: + case ValueOpcode::SelectU1: + if (ternary()) { + result = a != 0u ? b : c; + return true; + } + return false; + case ValueOpcode::IEqual32: + if (binary()) { + result = static_cast(a) == static_cast(b); + return true; + } + return false; + case ValueOpcode::INotEqual32: + if (binary()) { + result = static_cast(a) != static_cast(b); + return true; + } + return false; + case ValueOpcode::ULessThan32: + if (binary()) { + result = static_cast(a) < static_cast(b); + return true; + } + return false; + case ValueOpcode::UGreaterThan32: + if (binary()) { + result = static_cast(a) > static_cast(b); + return true; + } + return false; + case ValueOpcode::LogicalAnd: + if (binary()) { + result = (a != 0u) && (b != 0u); + return true; + } + return false; + case ValueOpcode::LogicalOr: + if (binary()) { + result = (a != 0u) || (b != 0u); + return true; + } + return false; + case ValueOpcode::LogicalXor: + if (binary()) { + result = (a != 0u) != (b != 0u); + return true; + } + return false; + case ValueOpcode::LogicalNot: + if (Arg(inst, 0, a, error)) { + result = a == 0u; + return true; + } + return false; + case ValueOpcode::UndefU1: + case ValueOpcode::UndefU8: + case ValueOpcode::UndefU16: + case ValueOpcode::UndefU32: + case ValueOpcode::UndefU64: return Fail(error, "undefined typed runtime value"); + default: break; + } + return Fail(error, fmt::format("unsupported typed runtime opcode {}", + ValueOpcodeName(inst.GetOpcode()))); + } + + const Program& m_program; + const ValueProgram& m_values; + const SrtRuntime& m_runtime; + uint32_t m_use_pc = 0; + std::unordered_map m_cache; + std::vector m_visiting; +}; + +const DescriptorSource* Source(const Program& program, uint32_t source) { + if (program.values == nullptr || source >= program.values->descriptor_sources.size()) { + return nullptr; + } + return &program.values->descriptor_sources[source]; +} + +bool EvaluateRuntimeSourcesImpl(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector& results, + std::vector& flat, bool evaluate_flat, + std::string* error) { + if (program.values == nullptr || !program.srt_plan_complete) { + if (error != nullptr) { + *error = Diagnostic(program, 0, "typed SRT plan is not ready"); + } + return false; + } + Evaluator evaluator(program, runtime); + std::vector evaluated; + evaluated.reserve(requests.size()); + for (const auto& request: requests) { + const auto* source = Source(program, request.source); + if (source == nullptr) { + if (error != nullptr) { + *error = Diagnostic(program, request.use_pc, + fmt::format("invalid descriptor source {}", request.source)); + } + return false; + } + evaluator.SetUsePc(request.use_pc); + DescriptorValue value; + value.dword_count = source->dword_count; + for (uint32_t index = 0; index < source->dword_count; index++) { + if (!evaluator.Evaluate(source->dwords[index], value.dwords[index], error)) { + return false; + } + } + evaluated.push_back(value); + } + std::vector flattened; + if (evaluate_flat) { + flattened.resize(program.values->srt_reads.size()); + for (const auto& read: program.values->srt_reads) { + evaluator.SetUsePc(read.use_pc); + if (read.flat_offset >= flattened.size() || + !evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) { + return false; + } + } + } + results = std::move(evaluated); + if (evaluate_flat) { + flat = std::move(flattened); + } + return true; +} + +} // namespace + +bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason) { + if (program.values == nullptr) { + reason = "typed SSA is not ready"; + return false; + } + return RuntimeValidator(program, *program.values).Run(value, reason); +} + +bool BuildSrtPlan(Program& program, std::string* error) { + if (program.values == nullptr || program.resource_tracking_complete) { + if (error != nullptr) { + *error = program.values == nullptr ? "typed SSA is not ready" + : "cannot rebuild SRT after resource tracking"; + } + return false; + } + program.srt_plan_complete = false; + if (!PlanBuilder(program, *program.values).Run(error)) { + return false; + } + program.srt_plan_complete = true; + return true; +} + +bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, + const SrtRuntime& runtime, DescriptorValue& result, + std::string* error) { + const DescriptorSourceRequest request {source, use_pc}; + std::vector results; + if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) { + return false; + } + result = results.front(); + return true; +} + +bool EvaluateDescriptorSources(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector& results, + std::string* error) { + std::vector ignored; + return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error); +} + +bool EvaluateRuntimeSources(const Program& program, + std::span requests, + const SrtRuntime& runtime, std::vector& results, + std::vector& flat, std::string* error) { + return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error); +} + +bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector& flat, + std::string* error) { + std::vector ignored; + return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error); +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h similarity index 96% rename from src/graphics/shader/recompiler/ir/SrtWalker.h rename to src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 1a8bdab73..c22dff069 100644 --- a/src/graphics/shader/recompiler/ir/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -8,6 +8,8 @@ namespace Libs::Graphics::ShaderRecompiler::IR { +class Value; + using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* value); struct SrtRuntime { @@ -23,11 +25,10 @@ struct DescriptorSourceRequest { uint32_t use_pc = 0; }; -bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t& result); - // Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots; // dynamic offsets remain explicit and are never assigned a fake slot. bool BuildSrtPlan(Program& program, std::string* error); +bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason); bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc, const SrtRuntime& runtime, DescriptorValue& result, diff --git a/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp new file mode 100644 index 000000000..96c1e521c --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp @@ -0,0 +1,377 @@ +#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" + +#include +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { +namespace { + +struct SccTag { + auto operator<=>(const SccTag&) const = default; +}; +struct ThreadBitScalarReg { + ScalarReg reg {}; + auto operator<=>(const ThreadBitScalarReg&) const = default; +}; +struct ExecTag { + auto operator<=>(const ExecTag&) const = default; +}; +struct ExecLoTag { + auto operator<=>(const ExecLoTag&) const = default; +}; +struct ExecHiTag { + auto operator<=>(const ExecHiTag&) const = default; +}; +struct VccTag { + auto operator<=>(const VccTag&) const = default; +}; +struct VccLoTag { + auto operator<=>(const VccLoTag&) const = default; +}; +struct VccHiTag { + auto operator<=>(const VccHiTag&) const = default; +}; +struct M0Tag { + auto operator<=>(const M0Tag&) const = default; +}; +struct GotoVariable { + uint32_t index = 0; + auto operator<=>(const GotoVariable&) const = default; +}; + +using Variable = std::variant; +using ValueMap = std::unordered_map; + +struct DefTable { + const Value& Get(Block* block, ScalarReg reg) { + if (RegIndex(reg) >= NumScalarRegs) { + EXIT("SSA scalar read index is out of range: %u", RegIndex(reg)); + } + return block->ssa_sreg_values[RegIndex(reg)]; + } + void Set(Block* block, ScalarReg reg, Value value) { + if (RegIndex(reg) >= NumScalarRegs) { + EXIT("SSA scalar write index is out of range: %u", RegIndex(reg)); + } + block->ssa_sreg_values[RegIndex(reg)] = value; + } + const Value& Get(Block* block, ThreadBitScalarReg value) { + EXIT_IF(RegIndex(value.reg) >= NumScalarRegs); + return block->ssa_thread_bit_sreg_values[RegIndex(value.reg)]; + } + void Set(Block* block, ThreadBitScalarReg value, Value definition) { + EXIT_IF(RegIndex(value.reg) >= NumScalarRegs); + block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition; + } + + const Value& Get(Block* block, VectorReg reg) { + EXIT_IF(RegIndex(reg) >= NumVectorRegs); + return block->ssa_vreg_values[RegIndex(reg)]; + } + void Set(Block* block, VectorReg reg, Value value) { + EXIT_IF(RegIndex(reg) >= NumVectorRegs); + block->ssa_vreg_values[RegIndex(reg)] = value; + } + const Value& Get(Block* block, GotoVariable value) { + return goto_variables[value.index][block]; + } + void Set(Block* block, GotoVariable value, Value definition) { + goto_variables[value.index][block] = definition; + } + + const Value& Get(Block* block, SccTag) { return scc[block]; } + void Set(Block* block, SccTag, Value value) { scc[block] = value; } + const Value& Get(Block* block, ExecTag) { return exec[block]; } + void Set(Block* block, ExecTag, Value value) { exec[block] = value; } + const Value& Get(Block* block, ExecLoTag) { return exec_lo[block]; } + void Set(Block* block, ExecLoTag, Value value) { exec_lo[block] = value; } + const Value& Get(Block* block, ExecHiTag) { return exec_hi[block]; } + void Set(Block* block, ExecHiTag, Value value) { exec_hi[block] = value; } + const Value& Get(Block* block, VccTag) { return vcc[block]; } + void Set(Block* block, VccTag, Value value) { vcc[block] = value; } + const Value& Get(Block* block, VccLoTag) { return vcc_lo[block]; } + void Set(Block* block, VccLoTag, Value value) { vcc_lo[block] = value; } + const Value& Get(Block* block, VccHiTag) { return vcc_hi[block]; } + void Set(Block* block, VccHiTag, Value value) { vcc_hi[block] = value; } + const Value& Get(Block* block, M0Tag) { return m0[block]; } + void Set(Block* block, M0Tag, Value value) { m0[block] = value; } + + ValueMap scc; + ValueMap exec; + ValueMap exec_lo; + ValueMap exec_hi; + ValueMap vcc; + ValueMap vcc_lo; + ValueMap vcc_hi; + ValueMap m0; + std::unordered_map goto_variables; +}; + +ValueOpcode UndefOpcode(ScalarReg) { + return ValueOpcode::UndefU32; +} +ValueOpcode UndefOpcode(ThreadBitScalarReg) { + return ValueOpcode::UndefU1; +} +ValueOpcode UndefOpcode(VectorReg) { + return ValueOpcode::UndefU32; +} +ValueOpcode UndefOpcode(GotoVariable) { + return ValueOpcode::UndefU1; +} +ValueOpcode UndefOpcode(SccTag) { + return ValueOpcode::UndefU1; +} +ValueOpcode UndefOpcode(ExecTag) { + return ValueOpcode::UndefU1; +} +ValueOpcode UndefOpcode(ExecLoTag) { + return ValueOpcode::UndefU32; +} +ValueOpcode UndefOpcode(ExecHiTag) { + return ValueOpcode::UndefU32; +} +ValueOpcode UndefOpcode(VccTag) { + return ValueOpcode::UndefU1; +} +ValueOpcode UndefOpcode(VccLoTag) { + return ValueOpcode::UndefU32; +} +ValueOpcode UndefOpcode(VccHiTag) { + return ValueOpcode::UndefU32; +} +ValueOpcode UndefOpcode(M0Tag) { + return ValueOpcode::UndefU32; +} + +Value InitialValue(ScalarReg) { + return Value(0u); +} +Value InitialValue(ThreadBitScalarReg) { + return Value(false); +} +Value InitialValue(VectorReg) { + return Value(0u); +} +Value InitialValue(GotoVariable) { + return Value(false); +} +Value InitialValue(SccTag) { + return Value(false); +} +Value InitialValue(ExecTag) { + return Value(false); +} +Value InitialValue(ExecLoTag) { + return Value(0u); +} +Value InitialValue(ExecHiTag) { + return Value(0u); +} +Value InitialValue(VccTag) { + return Value(false); +} +Value InitialValue(VccLoTag) { + return Value(0u); +} +Value InitialValue(VccHiTag) { + return Value(0u); +} +Value InitialValue(M0Tag) { + return Value(0u); +} + +enum class ReadStep { Start, SetValue, PushPhiArgument }; + +template +struct ReadState { + Block* block = nullptr; + Value result; + Inst* phi = nullptr; + size_t pred = 0; + ReadStep step = ReadStep::Start; +}; + +class Pass { +public: + template + void Write(T variable, Block* block, Value value) { + definitions.Set(block, variable, value); + } + + template + Value Read(T variable, Block* root) { + std::vector> stack {{}, {.block = root}}; + const auto prepare_phi = [&]() { + auto& state = stack.back(); + const auto predecessors = state.block->ImmPredecessors(); + if (state.pred == predecessors.size()) { + const auto result = TryRemoveTrivialPhi(*state.phi, InitialValue(variable)); + const auto block = state.block; + stack.pop_back(); + stack.back().result = result; + Write(variable, block, result); + return; + } + state.step = ReadStep::PushPhiArgument; + stack.push_back({.block = predecessors[state.pred]}); + }; + + do { + auto& state = stack.back(); + auto* block = state.block; + switch (state.step) { + case ReadStep::Start: { + if (const auto& def = definitions.Get(block, variable); !def.IsEmpty()) { + state.result = def; + } else if (!block->IsSsaSealed()) { + auto& phi = *block->PrependNewInst(block->begin(), ValueOpcode::Phi); + phi.SetFlags(TypeOf(UndefOpcode(variable))); + incomplete_phis[block][Variable(variable)] = φ + state.result = Value(&phi); + } else if (const auto predecessors = block->ImmPredecessors(); + predecessors.size() == 1) { + state.step = ReadStep::SetValue; + stack.push_back({.block = predecessors.front()}); + break; + } else { + auto& phi = *block->PrependNewInst(block->begin(), ValueOpcode::Phi); + phi.SetFlags(TypeOf(UndefOpcode(variable))); + Write(variable, block, Value(&phi)); + state.phi = φ + prepare_phi(); + break; + } + [[fallthrough]]; + } + case ReadStep::SetValue: { + const auto result = state.result; + Write(variable, block, result); + stack.pop_back(); + stack.back().result = result; + break; + } + case ReadStep::PushPhiArgument: { + const auto predecessors = block->ImmPredecessors(); + state.phi->AddPhiOperand(predecessors[state.pred], state.result); + state.pred++; + prepare_phi(); + break; + } + } + } while (stack.size() > 1); + return stack.back().result; + } + + void Seal(Block* block) { + if (const auto found = incomplete_phis.find(block); found != incomplete_phis.end()) { + for (auto& [variable, phi]: found->second) { + std::visit([&](auto value) { AddPhiOperands(value, *phi, block); }, variable); + } + } + block->SsaSeal(); + } + +private: + template + Value AddPhiOperands(T variable, Inst& phi, Block* block) { + for (auto* predecessor: block->ImmPredecessors()) { + phi.AddPhiOperand(predecessor, Read(variable, predecessor)); + } + return TryRemoveTrivialPhi(phi, InitialValue(variable)); + } + + Value TryRemoveTrivialPhi(Inst& phi, Value initial_value) { + Value same; + for (size_t index = 0; index < phi.NumArgs(); index++) { + const auto operand = phi.Arg(index).Resolve(); + if (operand == same.Resolve() || operand == Value(&phi)) { + continue; + } + if (!same.IsEmpty()) { + return Value(&phi); + } + same = operand; + } + if (same.IsEmpty()) { + same = initial_value; + } + const auto users = phi.Uses(); + phi.ReplaceUsesWith(same); + for (const auto& use: users) { + if (use.user->GetOpcode() == ValueOpcode::Phi) { + TryRemoveTrivialPhi(*use.user, initial_value); + } + } + return same; + } + + std::unordered_map> incomplete_phis; + DefTable definitions; +}; + +void VisitInstruction(Pass& pass, Block* block, Inst& inst) { + switch (inst.GetOpcode()) { + case ValueOpcode::SetScalarRegister: + pass.Write(inst.Arg(0).ScalarRegister(), block, inst.Arg(1)); + break; + case ValueOpcode::SetThreadBitScalarRegister: + pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1)); + break; + case ValueOpcode::SetVectorRegister: + pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1)); + break; + case ValueOpcode::SetGotoVariable: + pass.Write(GotoVariable {inst.Arg(0).U32()}, block, inst.Arg(1)); + break; + case ValueOpcode::SetScc: pass.Write(SccTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetExec: pass.Write(ExecTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetExecLo: pass.Write(ExecLoTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetExecHi: pass.Write(ExecHiTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetVcc: pass.Write(VccTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetVccLo: pass.Write(VccLoTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetVccHi: pass.Write(VccHiTag {}, block, inst.Arg(0)); break; + case ValueOpcode::SetM0: pass.Write(M0Tag {}, block, inst.Arg(0)); break; + case ValueOpcode::GetScalarRegister: + inst.ReplaceUsesWith(pass.Read(inst.Arg(0).ScalarRegister(), block)); + break; + case ValueOpcode::GetThreadBitScalarRegister: + inst.ReplaceUsesWith( + pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block)); + break; + case ValueOpcode::GetVectorRegister: + inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block)); + break; + case ValueOpcode::GetGotoVariable: + inst.ReplaceUsesWith(pass.Read(GotoVariable {inst.Arg(0).U32()}, block)); + break; + case ValueOpcode::GetScc: inst.ReplaceUsesWith(pass.Read(SccTag {}, block)); break; + case ValueOpcode::GetExec: inst.ReplaceUsesWith(pass.Read(ExecTag {}, block)); break; + case ValueOpcode::GetExecLo: inst.ReplaceUsesWith(pass.Read(ExecLoTag {}, block)); break; + case ValueOpcode::GetExecHi: inst.ReplaceUsesWith(pass.Read(ExecHiTag {}, block)); break; + case ValueOpcode::GetVcc: inst.ReplaceUsesWith(pass.Read(VccTag {}, block)); break; + case ValueOpcode::GetVccLo: inst.ReplaceUsesWith(pass.Read(VccLoTag {}, block)); break; + case ValueOpcode::GetVccHi: inst.ReplaceUsesWith(pass.Read(VccHiTag {}, block)); break; + case ValueOpcode::GetM0: inst.ReplaceUsesWith(pass.Read(M0Tag {}, block)); break; + default: break; + } +} + +} // namespace + +void RewriteToSsa(const BlockList& blocks) { + Pass pass; + for (auto* block: blocks) { + for (auto& inst: *block) { + VisitInstruction(pass, block, inst); + } + } + for (auto* block: blocks) { + pass.Seal(block); + } +} + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/SsaRewrite.h b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.h new file mode 100644 index 000000000..fbcc53822 --- /dev/null +++ b/src/graphics/shader/recompiler/ir/passes/SsaRewrite.h @@ -0,0 +1,9 @@ +#pragma once + +#include "graphics/shader/recompiler/ir/Block.h" + +namespace Libs::Graphics::ShaderRecompiler::IR { + +void RewriteToSsa(const BlockList& blocks); + +} // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/rectListShader.cpp b/src/graphics/shader/rectListShader.cpp index 3a7ad1c4e..b84f6f80d 100644 --- a/src/graphics/shader/rectListShader.cpp +++ b/src/graphics/shader/rectListShader.cpp @@ -1,7 +1,7 @@ #include "graphics/shader/rectListShader.h" #include "common/assert.h" -#include "graphics/shader/recompiler/emitter/SpirvBuilder.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shader.h" #include "spirv/unified1/spirv.hpp11" diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index cb9b20227..bf5ef7be0 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -14,7 +14,8 @@ #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/shaderVertexMetadata.h" #include "libs/errno.h" #include "spirv-tools/libspirv.h" @@ -1011,7 +1012,8 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog program.bindings.descriptor_set, program.bindings.push_constant_size, program.bindings.descriptors.size(), program.bindings.user_data_registers.size(), program.info.buffers.size(), program.info.images.size(), program.info.samplers.size(), - program.info.addresses.size(), program.srt.reads.size()); + program.info.addresses.size(), + program.values != nullptr ? program.values->srt_reads.size() : 0u); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { const auto& buffer = program.info.buffers[i]; ret += fmt::format(" b{}[stride={} format={}]", i, buffer.packed_stride, diff --git a/src/graphics/shader/shaderStageRuntime.cpp b/src/graphics/shader/shaderStageRuntime.cpp index 513e059a8..29bf5fd46 100644 --- a/src/graphics/shader/shaderStageRuntime.cpp +++ b/src/graphics/shader/shaderStageRuntime.cpp @@ -1,4 +1,4 @@ -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" #include "graphics/shader/shader.h" #include diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index dfaedce4e..1ef021e75 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -1,1958 +1,643 @@ #include "graphics/guest_gpu/gpu_defs.h" -#include "graphics/shader/recompiler/ir/BindingLayout.h" -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" -#include "graphics/shader/recompiler/ir/ResourceTracking.h" -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" -#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/SrtPatcher.h" -#include "graphics/shader/recompiler/ir/SrtWalker.h" -#include "graphics/shader/shaderBindings.h" +#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" +#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" -#include #include +#include #include +#include +#include #include +#include +#include namespace { using namespace Libs::Graphics::ShaderRecompiler::IR; using Libs::Graphics::ShaderComputeInputInfo; -using Libs::Graphics::ShaderPixelInputInfo; -using Libs::Graphics::ShaderTextureResource; using Libs::Graphics::ShaderType; -using Libs::Graphics::ShaderVertexInputInfo; -namespace Prospero = Libs::Graphics::Prospero; -namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; +namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; -void Check(bool condition, const char* message) { - if (!condition) { - throw std::runtime_error(message); - } +void Check(bool condition, const char *message) { + if (!condition) { + throw std::runtime_error(message); + } } -Operand Sgpr(uint32_t reg) { - Operand operand; - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Scalar; - operand.reg.index = reg; - return operand; -} +struct Fixture { + Program program; + Block *block = nullptr; -Operand Imm(uint32_t value) { - Operand operand; - operand.kind = OperandKind::ImmediateU32; - operand.imm = value; - return operand; -} + explicit Fixture(ShaderType stage = ShaderType::Compute) { + program.stage = stage; + program.values = std::make_shared(); + program.user_data_count = 64; + block = AddBlock(); + } -Instruction Move(uint32_t pc, uint32_t dst, uint32_t src) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::MoveU32; - inst.dst = Sgpr(dst); - inst.src[0] = Sgpr(src); - inst.src_count = 1; - return inst; -} + Block *AddBlock() { + auto storage = std::make_unique(); + auto *result = storage.get(); + program.values->block_storage.push_back(std::move(storage)); + program.values->blocks.push_back(result); + program.values->block_info.push_back( + {.id = static_cast(program.values->block_info.size())}); + return result; + } -Instruction MoveImmediate(uint32_t pc, uint32_t dst, uint32_t value) { - auto inst = Move(pc, dst, 0); - inst.src[0] = Imm(value); - return inst; -} + Value Emit(ValueOpcode opcode, std::initializer_list args = {}, + uint64_t flags = 0, Block *destination = nullptr) { + if (NumArgsOf(opcode) != std::numeric_limits::max() && + NumArgsOf(opcode) != args.size()) { + throw std::runtime_error(std::string(ValueOpcodeName(opcode)) + + " argument count"); + } + auto &inst = (destination != nullptr ? destination : block) + ->AppendNewInst(opcode, args, flags); + return Value(&inst); + } -Instruction BufferUse(uint32_t pc, uint32_t base, Opcode op = Opcode::BufferLoadDword) { - Instruction inst; - inst.pc = pc; - inst.op = op; - inst.memory.kind = ResourceKind::Buffer; - inst.memory.resource = base / 4; - return inst; -} + template + Value Emit(ValueOpcode opcode, std::initializer_list args, T flags, + Block *destination = nullptr) { + uint64_t bits = 0; + std::memcpy(&bits, &flags, sizeof(flags)); + return Emit(opcode, args, bits, destination); + } -Instruction ImageUse(uint32_t pc, Opcode op, ResourceKind kind, Decoder::ImageDimension dimension, - uint32_t resource = 0, uint32_t sampler = 2) { - Instruction inst; - inst.pc = pc; - inst.op = op; - inst.memory.kind = kind; - inst.memory.resource = resource; - inst.memory.sampler = sampler; - inst.memory.image_dimension = dimension; - return inst; -} + Value UserData(uint32_t index) { + return Emit(ValueOpcode::GetUserData, + {Value(static_cast(index))}); + } -Instruction ScalarLoad(uint32_t pc, uint32_t dst, uint32_t base, uint32_t offset) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::SLoadDword; - inst.dst = Sgpr(dst); - inst.memory.kind = ResourceKind::ScalarBuffer; - inst.memory.resource = base; - inst.memory.offset = offset; - return inst; -} + MemoryFlags AddMemory(MemoryInfo memory, uint32_t pc) { + const auto index = + static_cast(program.values->memory_info.size()); + program.values->memory_info.push_back(memory); + return {index, pc}; + } -Instruction ScalarBufferLoad(uint32_t pc, uint32_t dst, uint32_t descriptor, uint32_t offset) { - auto inst = ScalarLoad(pc, dst, descriptor * 4, offset); - inst.op = Opcode::SBufferLoadDword; - inst.memory.resource = descriptor; - return inst; -} + Value Buffer(std::array dwords, uint32_t pc = 0) { + return Emit(ValueOpcode::GetBufferResource, + {dwords[0], dwords[1], dwords[2], dwords[3]}, + MemoryFlags{0, pc}); + } -void MakeScalarMemoryGroup(std::vector* instructions) { - const auto count = static_cast(instructions->size()); - for (uint32_t i = 0; i < count; i++) { - (*instructions)[i].memory.component_index = i; - (*instructions)[i].memory.component_count = count; - } -} + Value Address(Value low, Value high, uint32_t pc = 0) { + return Emit(ValueOpcode::GetAddressResource, {low, high}, + MemoryFlags{0, pc}); + } -Instruction Export(uint32_t pc, ExportTargetKind kind, uint32_t index = 0, uint32_t en = 0xf) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::Export; - inst.export_info.kind = kind; - inst.export_info.index = index; - inst.export_info.en = en; - return inst; -} + Value Image(std::array dwords, uint32_t pc = 0) { + return Emit(ValueOpcode::GetImageResource, + {dwords[0], dwords[1], dwords[2], dwords[3], dwords[4], + dwords[5], dwords[6], dwords[7]}, + MemoryFlags{0, pc}); + } -struct TestMemory { - uint64_t base = 0x1000; - std::array words {}; - uint32_t reads = 0; - uint32_t fail_after = UINT32_MAX; + Value Sampler(std::array dwords, uint32_t pc = 0) { + return Emit(ValueOpcode::GetSamplerResource, + {dwords[0], dwords[1], dwords[2], dwords[3]}, + MemoryFlags{0, pc}); + } + + Value ImageAddress() { + return Emit(ValueOpcode::MakeImageAddress, + {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u)}); + } + + void PlanAndTrack() { + std::string error; + if (!BuildSrtPlan(program, &error) || !TrackResources(program, &error)) { + throw std::runtime_error(error); + } + } }; -bool ReadTestMemory(void* userdata, uint64_t address, uint32_t* value) { - auto* memory = static_cast(userdata); - if (memory == nullptr || value == nullptr || address < memory->base || - address - memory->base >= memory->words.size() * sizeof(uint32_t) || - memory->reads >= memory->fail_after) { - return false; - } - const auto index = static_cast((address - memory->base) / sizeof(uint32_t)); - *value = memory->words[index]; - memory->reads++; - return true; +struct TestMemory { + uint64_t base = 0x1000; + std::array words{}; + uint32_t reads = 0; + uint32_t fail_after = UINT32_MAX; +}; + +bool ReadTestMemory(void *userdata, uint64_t address, uint32_t *value) { + auto *memory = static_cast(userdata); + if (memory == nullptr || value == nullptr || address < memory->base || + address - memory->base >= memory->words.size() * sizeof(uint32_t) || + memory->reads >= memory->fail_after) { + return false; + } + *value = memory->words[(address - memory->base) / sizeof(uint32_t)]; + memory->reads++; + return true; } -const StageInput* FindInput(const ShaderInfo& info, StageInputKind kind, uint32_t location = 0) { - for (const auto& input: info.inputs) { - if (input.kind == kind && input.location == location) { - return &input; - } - } - return nullptr; -} +void TestDenseBufferTracking() { + Fixture fixture; + std::array userdata; + for (uint32_t index = 0; index < userdata.size(); index++) { + userdata[index] = fixture.UserData(index); + } + const auto first = + fixture.Buffer({userdata[0], userdata[1], userdata[2], userdata[3]}, 4); + const auto second = + fixture.Buffer({userdata[4], userdata[5], userdata[6], userdata[7]}, 28); -const StageOutput* FindOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index = 0) { - for (const auto& output: info.outputs) { - if (output.kind == kind && output.index == index) { - return &output; - } - } - return nullptr; -} + MemoryInfo load_info; + load_info.kind = ResourceKind::Buffer; + load_info.offset = 4; + load_info.formatted = true; + const auto load_flags = fixture.AddMemory(load_info, 4); + fixture.Emit(ValueOpcode::LoadBufferU32, + {first, Value(0u), Value(0u), Value(0u), Value(true)}, + load_flags); -void Prepare(Program& program) { - std::string error; - if (!BuildScalarProvenance(program, &error) || !BuildSrtPlan(program, &error) || - !PatchSrtReads(program, &error) || !TrackResources(program, &error)) { - throw std::runtime_error(error); - } -} + auto store_info = load_info; + store_info.offset = 12; + const auto store_flags = fixture.AddMemory(store_info, 8); + fixture.Emit(ValueOpcode::StoreBufferU32, + {first, Value(0u), Value(0u), Value(0u), Value(7u), Value(true)}, + store_flags); -void TestDenseBufferPatching() { - Program program; - program.blocks.resize(1); - auto first = BufferUse(4, 0); - first.memory.offset = 4; - first.memory.formatted = true; - auto write = BufferUse(8, 0, Opcode::BufferStoreDword); - write.memory.offset = 12; - auto atomic = BufferUse(10, 0, Opcode::AtomicAddU32); - auto& insts = program.blocks[0].instructions; - insts = {first, atomic, write, Move(12, 4, 20), - Move(16, 5, 21), Move(20, 6, 22), Move(24, 7, 23), BufferUse(28, 4)}; + auto atomic_info = load_info; + atomic_info.offset = 0; + const auto atomic_flags = fixture.AddMemory(atomic_info, 12); + fixture.Emit(ValueOpcode::BufferAtomicIAdd32, + {first, Value(0u), Value(0u), Value(1u), Value(0u), Value(true)}, + atomic_flags); - Prepare(program); - Check(program.info.buffers.size() == 2, "buffer sources were not densely deduplicated"); - const auto& resource = program.info.buffers[0]; - Check(resource.read && resource.written && resource.atomic && resource.formatted && - resource.max_byte_extent == 16 && resource.first_use_pc == 4, - "buffer access facts were not merged"); - Check(insts[0].memory.resource == 0 && insts[1].memory.resource == 0 && - insts[2].memory.resource == 0 && insts[7].memory.resource == 1, - "buffer operands were not patched to dense indices"); - Check(insts[0].memory.resource_source == ScalarProvenance::Undefined && - insts[7].memory.resource_source == ScalarProvenance::Undefined, - "patched instructions retained duplicate descriptor source handles"); - std::string error; - Check(!TrackResources(program, &error) && error.find("already tracked") != std::string::npos, - "resource tracking was not guarded against a second patch pass"); + const auto other_flags = fixture.AddMemory(load_info, 28); + fixture.Emit(ValueOpcode::LoadBufferU32, + {second, Value(0u), Value(0u), Value(0u), Value(true)}, + other_flags); + fixture.PlanAndTrack(); + + Check(fixture.program.info.buffers.size() == 2, + "typed buffer sources were not densely interned"); + Check(fixture.program.values->descriptor_sources.size() == 2, + "descriptor source table did not match dense topology"); + const auto &resource = fixture.program.info.buffers[0]; + Check(resource.read && resource.written && resource.atomic && + resource.formatted && resource.max_byte_extent == 16 && + resource.first_use_pc == 4, + "buffer access facts were not merged"); + Check(first.Instruction()->Flags() == 0 && + second.Instruction()->Flags() == 1, + "typed handles were not assigned dense indices"); + Check(fixture.program.values->memory_info[load_flags.index].resource == 0 && + fixture.program.values->memory_info[store_flags.index].resource == + 0 && + fixture.program.values->memory_info[other_flags.index].resource == + 1, + "typed memory metadata was not patched to dense indices"); + + std::string error; + Check(!TrackResources(fixture.program, &error) && + error.find("already tracked") != std::string::npos, + "resource tracking allowed a second mutation pass"); } void TestScalarAndVectorBufferAlias() { - Program program; - program.blocks.resize(1); - Instruction scalar; - scalar.pc = 4; - scalar.op = Opcode::SBufferLoadDword; - scalar.dst = Sgpr(20); - scalar.src[0] = Imm(0); - scalar.src_count = 1; - scalar.memory.kind = ResourceKind::ScalarBuffer; - scalar.memory.resource = 2; - program.blocks[0].instructions = {scalar, BufferUse(8, 8)}; + Fixture fixture; + const auto d0 = fixture.UserData(0); + const auto d1 = fixture.UserData(1); + const auto d2 = fixture.UserData(2); + const auto d3 = fixture.UserData(3); + const auto descriptor = fixture.Buffer({d0, d1, d2, d3}, 4); - Prepare(program); - Check(program.info.buffers.size() == 1 && program.info.buffers[0].scalar, - "scalar and vector uses of one descriptor were split"); - Check(program.blocks[0].instructions[0].memory.resource == 0 && - program.blocks[0].instructions[1].memory.resource == 0, - "scalar/vector alias did not share one dense index"); + MemoryInfo scalar; + scalar.kind = ResourceKind::ScalarBuffer; + const auto scalar_flags = fixture.AddMemory(scalar, 4); + fixture.Emit(ValueOpcode::ReadConstBuffer, {descriptor, fixture.UserData(4)}, + scalar_flags); + MemoryInfo vector; + vector.kind = ResourceKind::Buffer; + const auto vector_flags = fixture.AddMemory(vector, 8); + fixture.Emit(ValueOpcode::LoadBufferU32, + {descriptor, Value(0u), Value(0u), Value(0u), Value(true)}, + vector_flags); + fixture.PlanAndTrack(); + + Check(fixture.program.info.buffers.size() == 1 && + fixture.program.info.buffers[0].scalar, + "typed scalar and vector uses of one descriptor were split"); + Check(fixture.program.values->memory_info[scalar_flags.index].resource == 0 && + fixture.program.values->memory_info[vector_flags.index].resource == + 0, + "scalar/vector alias did not share a dense index"); } -void TestBufferImageAliasIsLinkedDuringTracking() { - Program program; - program.blocks.resize(1); - program.blocks[0].instructions = { - BufferUse(4, 0), BufferUse(8, 8), - ImageUse(12, Opcode::ImageLoad, ResourceKind::Image, Decoder::ImageDimension::Dim2D)}; +void TestImagesSamplersAndAliases() { + Fixture fixture; + std::array image_words; + for (uint32_t index = 0; index < image_words.size(); index++) { + image_words[index] = fixture.UserData(index); + } + const auto image_address = fixture.ImageAddress(); + const std::array sampler0{Value(0u), Value(1u), Value(2u), + Value(0x1111u)}; + const std::array sampler1{Value(0u), Value(1u), Value(2u), + Value(0x2222u)}; - Prepare(program); - Check(program.info.buffers.size() == 2 && program.info.images.size() == 1 && - program.info.buffers[0].image_alias == 0 && - program.info.buffers[1].image_alias == BufferResource::NoImageAlias, - "buffer/image descriptor provenance aliases were not linked during " - "tracking"); + auto AddSample = [&](uint32_t pc, uint32_t sample_flags, + const auto &sampler_words) { + const auto image = fixture.Image(image_words, pc); + const auto sampler = fixture.Sampler(sampler_words, pc); + MemoryInfo memory; + memory.kind = ResourceKind::Image; + memory.image_dimension = Decoder::ImageDimension::Dim2D; + memory.image_sample_flags = sample_flags; + fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, image_address}, + fixture.AddMemory(memory, pc)); + return std::pair{image, sampler}; + }; + const auto normal = AddSample(4, 0, sampler0); + const auto repeated = AddSample(8, 0, sampler1); + const auto compare = AddSample(12, Decoder::ImageSampleFlagCompare, sampler0); + + const auto storage = fixture.Image(image_words, 16); + MemoryInfo storage_memory; + storage_memory.kind = ResourceKind::StorageImage; + storage_memory.image_dimension = Decoder::ImageDimension::Dim2D; + fixture.Emit(ValueOpcode::ImageAtomicIAdd32, + {storage, image_address, Value(1u), Value(true)}, + fixture.AddMemory(storage_memory, 16)); + + const auto buffer = fixture.Buffer( + {image_words[0], image_words[1], image_words[2], image_words[3]}, 20); + MemoryInfo buffer_memory; + buffer_memory.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {buffer, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(buffer_memory, 20)); + fixture.PlanAndTrack(); + + Check(fixture.program.info.images.size() == 3 && + fixture.program.info.samplers.size() == 1 && + fixture.program.info.sampled_pairs.size() == 2, + "typed image view classes or samplers were deduplicated incorrectly"); + Check(normal.first.Instruction()->Flags() == + repeated.first.Instruction()->Flags() && + compare.first.Instruction()->Flags() != + normal.first.Instruction()->Flags(), + "image handles did not receive view-class indices"); + Check(normal.second.Instruction()->Flags() == 0 && + repeated.second.Instruction()->Flags() == 0, + "unused sampler border colors prevented source interning"); + const auto sampler_source = fixture.program.info.samplers[0].source; + Check(fixture.program.values->descriptor_sources[sampler_source] + .dwords[3] + .U32() == 0, + "unused sampler border color was not canonicalized"); + Check(fixture.program.info.buffers[0].image_alias == 0, + "buffer/image descriptor alias was not linked"); } -void TestImagesAndSamplers() { - Program program; - program.blocks.resize(1); - auto sample0 = - ImageUse(4, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D); - auto sample1 = - ImageUse(8, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D); - auto volume = - ImageUse(12, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim3D); - auto compare = - ImageUse(16, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D); - compare.memory.image_sample_flags = Decoder::ImageSampleFlagCompare; - auto storage = ImageUse(20, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D); - auto storage_mip = storage; - storage_mip.pc = 24; - storage_mip.memory.image_has_mip = true; - auto atomic = ImageUse(28, Opcode::AtomicAddU32, ResourceKind::StorageImageUint, - Decoder::ImageDimension::Dim2D); - program.blocks[0].instructions = {sample0, sample1, volume, compare, - storage, storage_mip, atomic}; +void TestSrtFlatteningAndRuntimeMemoization() { + Fixture fixture; + const auto base = + fixture.Address(fixture.UserData(0), fixture.UserData(1), 4); + MemoryInfo scalar; + scalar.kind = ResourceKind::ScalarBuffer; + scalar.offset = 4; + const auto read0 = fixture.Emit(ValueOpcode::LoadAddressU32, + {base, Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(scalar, 4)); + const auto descriptor0 = + fixture.Buffer({read0, Value(0u), Value(64u), Value(0u)}, 12); + const auto descriptor1 = + fixture.Buffer({read0, Value(0u), Value(64u), Value(0u)}, 16); + MemoryInfo buffer; + buffer.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {descriptor0, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(buffer, 12)); + fixture.Emit(ValueOpcode::LoadBufferU32, + {descriptor1, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(buffer, 16)); + fixture.PlanAndTrack(); - Prepare(program); - Check(program.info.images.size() == 6 && program.info.samplers.size() == 1 && - program.info.sampled_pairs.size() == 3, - "image view classes or samplers were deduplicated incorrectly"); - const auto& insts = program.blocks[0].instructions; - Check(insts[0].memory.resource == 0 && insts[1].memory.resource == 0 && - insts[2].memory.resource == 1 && insts[3].memory.resource == 2 && - insts[4].memory.resource == 3 && insts[5].memory.resource == 4 && - insts[6].memory.resource == 5 && insts[0].memory.sampler == 0 && - insts[2].memory.sampler == 0, - "image/sampler operands were not patched to dense list indices"); - Check(program.info.images[0].read && !program.info.images[0].written && - program.info.images[2].depth_compare && program.info.images[3].written && - program.info.images[4].mip_mode == ImageMipMode::DynamicStorage && - program.info.images[5].atomic && - program.info.sampled_pairs[0].sampler == program.info.sampled_pairs[1].sampler, - "image access facts were wrong"); -} + Check(fixture.program.values->srt_reads.size() == 1, + "shared typed scalar read did not receive one flat SRT slot"); + Check(fixture.program.info.buffers.size() == 1 && + fixture.program.info.addresses.empty(), + "planning-only scalar reads leaked into resource topology"); + Check(fixture.program.values->memory_info[0].planning_only, + "canonical runtime scalar read was not marked planning-only"); -void TestDynamicPhiResource() { - Program program; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - Instruction left; - left.op = Opcode::MoveU32; - left.dst = Sgpr(0); - left.src[0] = Imm(1); - left.src_count = 1; - Instruction right = left; - right.pc = 4; - right.src[0] = Imm(2); - program.blocks[1].instructions = {left}; - program.blocks[2].instructions = {right}; - program.blocks[3].instructions = {BufferUse(8, 0)}; + std::array user_data{0x1000u, 0u}; + TestMemory memory; + memory.words[1] = 0xdeadbeefu; + SrtRuntime runtime{.user_data = user_data, + .read_memory = ReadTestMemory, + .userdata = &memory}; + std::vector descriptors; + std::vector flat; + const DescriptorSourceRequest request{fixture.program.info.buffers[0].source, + 12}; + std::string error; + Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, + descriptors, flat, &error), + "typed runtime source evaluation failed"); + Check(descriptors.size() == 1 && descriptors[0].dwords[0] == 0xdeadbeefu && + flat == std::vector{0xdeadbeefu} && memory.reads == 1, + "descriptor and flat SRT evaluation did not share one memoized read"); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error), - error.c_str()); - const auto original_source = program.blocks[3].instructions[0].memory.resource_source; - Check(!TrackResources(program, &error) && - error.find("unsupported GPU selection") != std::string::npos && - program.blocks[3].instructions[0].memory.resource_source == original_source && - !program.resource_tracking_complete, - "control-flow descriptor was patched without an executable GPU selector"); -} + memory.reads = 0; + memory.fail_after = 0; + descriptors = {{{1u}, 1u}}; + flat = {2u}; + Check(!EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, + runtime, descriptors, flat, &error) && + descriptors == std::vector{{{1u}, 1u}} && + flat == std::vector{2u}, + "runtime evaluation failure was not transactional"); -void TestTrackingRequiresCompletedSrtPlan() { - Program program; - program.stage = ShaderType::Compute; - program.shader_hash = 0x8899; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - program.blocks[1].instructions = {MoveImmediate(0, 0, 1)}; - program.blocks[2].instructions = {MoveImmediate(4, 0, 2)}; - program.blocks[3].instructions = {BufferUse(8, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - Check(!program.srt_plan_complete, "provenance unexpectedly marked the SRT plan complete"); - const auto original_source = program.blocks[3].instructions[0].memory.resource_source; - BufferResource existing_info; - existing_info.source = 777; - program.info.buffers.push_back(existing_info); - Check(!TrackResources(program, &error) && - error.find("SRT plan is not ready") != std::string::npos && - program.blocks[3].instructions[0].memory.resource == 0 && - program.blocks[3].instructions[0].memory.resource_source == original_source && - program.info.buffers.size() == 1 && program.info.buffers[0].source == 777 && - !program.resource_tracking_complete, - "tracking bypassed the SRT-plan readiness invariant or partially " - "patched the program"); -} - -void TestCyclicResourceIsRejected() { - Program program; - program.blocks.resize(2); - program.blocks[0].successors = {1}; - program.blocks[1].predecessors = {0, 1}; - program.blocks[1].successors = {1}; - Instruction increment; - increment.op = Opcode::IAddU32; - increment.dst = Sgpr(0); - increment.src[0] = Sgpr(0); - increment.src[1] = Imm(1); - increment.src_count = 2; - program.blocks[1].instructions = {BufferUse(4, 0), increment}; - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error), - error.c_str()); - Check(!TrackResources(program, &error) && - error.find("unsupported GPU selection") != std::string::npos && - !program.resource_tracking_complete, - "cyclic descriptor was patched without a bindless/direct path"); -} - -void TestUnknownSourceFailsWithoutPatching() { - Program program; - program.stage = ShaderType::Pixel; - program.shader_hash = 0x12345678; - program.blocks.resize(1); - auto valid = BufferUse(4, 0); - Instruction unsupported; - unsupported.op = Opcode::SelectU32; - unsupported.dst = Sgpr(4); - unsupported.src[0] = Sgpr(20); - unsupported.src_count = 1; - program.blocks[0].instructions = {valid, unsupported, BufferUse(0x44, 4)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error), - error.c_str()); - BufferResource existing_info; - existing_info.source = 777; - program.info.buffers.push_back(existing_info); - Check(!TrackResources(program, &error), "unknown descriptor unexpectedly tracked"); - Check(error.find("hash=0x0000000012345678") != std::string::npos && - error.find("stage=pixel") != std::string::npos && - error.find("pc=0x00000044") != std::string::npos && - error.find("unknown value") != std::string::npos, - "unknown descriptor error lost shader context"); - Check(!program.resource_tracking_complete && program.info.buffers.size() == 1 && - program.info.buffers[0].source == 777 && - program.blocks[0].instructions[0].memory.resource == 0 && - program.blocks[0].instructions[0].memory.resource_source != - ScalarProvenance::Undefined && - program.blocks[0].instructions[2].memory.resource == 1 && - program.blocks[0].instructions[2].memory.resource_source != - ScalarProvenance::Undefined, - "failed tracking partially patched the program"); -} - -void TestResourceLimitFailsTransactionally() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i <= ShaderInfo::MaxBuffers; i++) { - insts.push_back(MoveImmediate(i * 8, 0, i)); - insts.push_back(BufferUse(i * 8 + 4, 0)); - } - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error), - error.c_str()); - const auto first_source = insts[1].memory.resource_source; - Check(!TrackResources(program, &error) && - error.find("buffer resource limit exceeded") != std::string::npos && - insts[1].memory.resource_source == first_source && program.info.buffers.empty() && - !program.resource_tracking_complete, - "resource limit failure partially patched the program"); -} - -void TestComputeShaderInfoCollection() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - Instruction add_tid; - add_tid.op = Opcode::DsReadAddtidB32; - program.blocks[0].instructions = {add_tid, BufferUse(4, 0)}; - Prepare(program); - const auto buffers = program.info.buffers; - const auto images = program.info.images; - const auto samplers = program.info.samplers; - const auto pairs = program.info.sampled_pairs; - - ShaderComputeInputInfo compute; - compute.group_id[1] = true; - compute.dispatch_thread_dimensions = true; - std::string error; - Check(CollectShaderInfo(program, {.compute = &compute}, &error), error.c_str()); - Check(program.shader_info_complete && program.info.inputs.size() == 3 && - FindInput(program.info, StageInputKind::WorkgroupId) != nullptr && - FindInput(program.info, StageInputKind::LocalInvocationId) == nullptr && - FindInput(program.info, StageInputKind::LocalInvocationIndex) != nullptr && - FindInput(program.info, StageInputKind::GlobalInvocationId) != nullptr && - program.info.buffers == buffers && program.info.images == images && - program.info.samplers == samplers && program.info.sampled_pairs == pairs, - "compute addtid input discovery or resource preservation was wrong"); - - Program metadata_program; - metadata_program.stage = ShaderType::Compute; - metadata_program.blocks.resize(1); - Instruction xor_address; - xor_address.op = Opcode::BitwiseXor3U32; - metadata_program.blocks[0].instructions = {xor_address}; - Prepare(metadata_program); - compute = {}; - compute.thread_ids_num = 2; - Check(CollectShaderInfo(metadata_program, {.compute = &compute}, &error), error.c_str()); - Check(FindInput(metadata_program.info, StageInputKind::LocalInvocationId) != nullptr && - FindInput(metadata_program.info, StageInputKind::LocalInvocationIndex) != nullptr && - metadata_program.info.has_bitwise_xor, - "compute thread metadata or XOR-address heuristic was not collected"); -} - -void TestVertexShaderInfoCollection() { - Program program; - program.stage = ShaderType::Vertex; - program.blocks.resize(1); - Instruction attr0; - attr0.op = Opcode::LoadInputF32; - attr0.input_info.attr = 0; - attr0.input_info.chan = 2; - Instruction attr2 = attr0; - attr2.input_info.attr = 2; - attr2.input_info.chan = 0; - program.blocks[0].instructions = {attr0, attr2, Export(8, ExportTargetKind::Position), - Export(12, ExportTargetKind::Parameter, 2)}; - Prepare(program); - - ShaderVertexInputInfo vertex; - vertex.resources_num = 3; - std::string error; - Check(CollectShaderInfo(program, {.vertex = &vertex}, &error), error.c_str()); - const auto* input0 = FindInput(program.info, StageInputKind::Parameter, 0); - const auto* input2 = FindInput(program.info, StageInputKind::Parameter, 2); - Check(program.info.inputs.size() == 4 && input0 != nullptr && input0->component_count == 3 && - input2 != nullptr && input2->component_count == 1 && - FindInput(program.info, StageInputKind::VertexIndex) != nullptr && - FindInput(program.info, StageInputKind::InstanceIndex) != nullptr && - FindOutput(program.info, StageOutputKind::Position) != nullptr && - FindOutput(program.info, StageOutputKind::Parameter, 2) != nullptr, - "vertex inputs or exports were not collected from lowered IR"); - - const auto info = program.info; - const auto resources_complete = program.resource_tracking_complete; - const auto srt_complete = program.srt_plan_complete; - const auto patching_complete = program.srt_patching_complete; - Check(!CollectShaderInfo(program, {.vertex = &vertex}, &error) && - error.find("already collected") != std::string::npos && program.info == info && - program.shader_info_complete && - program.resource_tracking_complete == resources_complete && - program.srt_plan_complete == srt_complete && - program.srt_patching_complete == patching_complete, - "repeated shader info collection mutated the immutable interface"); -} - -void TestPixelShaderInfoCollection() { - Program program; - program.stage = ShaderType::Pixel; - program.blocks.resize(1); - program.blocks[0].instructions = { - Export(4, ExportTargetKind::Mrt, 3), Export(8, ExportTargetKind::MrtZ, 0, 0x4), - Export(12, ExportTargetKind::Mrt, 3), Export(16, ExportTargetKind::Mrt, 7, 0)}; - Prepare(program); - - ShaderPixelInputInfo pixel; - pixel.ps_pos_x = true; - pixel.ps_front_face = true; - pixel.input_num = 2; - pixel.ps_depth_export_enable = true; - pixel.ps_sample_mask_export_enable = true; - std::string error; - Check(CollectShaderInfo(program, {.pixel = &pixel}, &error), error.c_str()); - Check(program.info.inputs.size() == 4 && - FindInput(program.info, StageInputKind::FragCoord) != nullptr && - FindInput(program.info, StageInputKind::FrontFacing) != nullptr && - FindInput(program.info, StageInputKind::Parameter, 0) != nullptr && - FindInput(program.info, StageInputKind::Parameter, 1) != nullptr && - program.info.outputs.size() == 2 && - FindOutput(program.info, StageOutputKind::Mrt, 3) != nullptr && - FindOutput(program.info, StageOutputKind::Mrt, 7) == nullptr && - FindOutput(program.info, StageOutputKind::Depth) == nullptr && - FindOutput(program.info, StageOutputKind::SampleMask) != nullptr, - "pixel inputs, disabled exports, or sample-mask-only MRTZ collection " - "was wrong"); - - Program depth_program; - depth_program.stage = ShaderType::Pixel; - depth_program.blocks.resize(1); - depth_program.blocks[0].instructions = {Export(4, ExportTargetKind::MrtZ, 0, 0x1)}; - Prepare(depth_program); - Check(CollectShaderInfo(depth_program, {.pixel = &pixel}, &error), error.c_str()); - Check(depth_program.info.outputs.size() == 1 && - FindOutput(depth_program.info, StageOutputKind::Depth) != nullptr && - FindOutput(depth_program.info, StageOutputKind::SampleMask) == nullptr, - "depth-only MRTZ export incorrectly enabled sample-mask output"); -} - -void TestShaderInfoCollectionIsTransactional() { - Program program; - program.stage = ShaderType::Compute; - StageInput sentinel; - sentinel.kind = StageInputKind::Parameter; - sentinel.location = 9; - program.info.inputs.push_back(sentinel); - const auto info = program.info; - std::string error; - Check(!CollectShaderInfo(program, {}, &error) && - error.find("not tracked") != std::string::npos && program.info == info && - !program.shader_info_complete, - "pre-track shader info collection mutated program state"); - - Prepare(program); - const auto tracked_info = program.info; - const auto srt_complete = program.srt_plan_complete; - const auto patching_complete = program.srt_patching_complete; - Check(!CollectShaderInfo(program, {}, &error) && - error.find("requires compute metadata") != std::string::npos && - program.info == tracked_info && !program.shader_info_complete && - program.resource_tracking_complete && program.srt_plan_complete == srt_complete && - program.srt_patching_complete == patching_complete, - "missing stage metadata committed incomplete shader info"); - program.stage = ShaderType::Unknown; - Check(!CollectShaderInfo(program, {}, &error) && - error.find("unsupported shader stage") != std::string::npos && - program.info == tracked_info && !program.shader_info_complete && - program.resource_tracking_complete && program.srt_plan_complete == srt_complete && - program.srt_patching_complete == patching_complete, - "unsupported-stage shader info collection was not transactional"); -} - -void TestShaderInfoMetadataValidation() { - std::string error; - - Program vertex_program; - vertex_program.stage = ShaderType::Vertex; - vertex_program.blocks.resize(1); - Instruction bad_input; - bad_input.op = Opcode::LoadInputF32; - bad_input.input_info.attr = 0; - bad_input.input_info.chan = 4; - vertex_program.blocks[0].instructions = {bad_input}; - Prepare(vertex_program); - ShaderVertexInputInfo vertex; - vertex.resources_num = 1; - const auto vertex_info = vertex_program.info; - Check(!CollectShaderInfo(vertex_program, {.vertex = &vertex}, &error) && - error.find("vertex input reference") != std::string::npos && - vertex_program.info == vertex_info && !vertex_program.shader_info_complete, - "out-of-range vertex channel produced immutable malformed info"); - vertex.resources_num = -1; - Check(!CollectShaderInfo(vertex_program, {.vertex = &vertex}, &error) && - error.find("vertex resource count") != std::string::npos && - vertex_program.info == vertex_info && !vertex_program.shader_info_complete, - "negative vertex resource count produced immutable malformed info"); - - Program pixel_program; - pixel_program.stage = ShaderType::Pixel; - pixel_program.blocks.resize(1); - Prepare(pixel_program); - ShaderPixelInputInfo pixel; - pixel.input_num = 33; - const auto pixel_info = pixel_program.info; - Check(!CollectShaderInfo(pixel_program, {.pixel = &pixel}, &error) && - error.find("pixel input count") != std::string::npos && - pixel_program.info == pixel_info && !pixel_program.shader_info_complete, - "out-of-range pixel input count produced immutable malformed info"); - - Program compute_program; - compute_program.stage = ShaderType::Compute; - compute_program.blocks.resize(1); - Prepare(compute_program); - ShaderComputeInputInfo compute; - compute.thread_ids_num = 4; - const auto compute_info = compute_program.info; - Check(!CollectShaderInfo(compute_program, {.compute = &compute}, &error) && - error.find("thread ID count") != std::string::npos && - compute_program.info == compute_info && !compute_program.shader_info_complete, - "out-of-range compute metadata produced immutable malformed info"); -} - -void TestTrackingRequiresSrtPatching() { - Program program; - program.blocks.resize(1); - program.blocks[0].instructions = {BufferUse(4, 0)}; - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const auto source = program.blocks[0].instructions[0].memory.resource_source; - Check(!TrackResources(program, &error) && - error.find("SRT reads were not patched") != std::string::npos && - program.blocks[0].instructions[0].memory.resource_source == source && - !program.resource_tracking_complete, - "resource tracking bypassed SRT patch completion"); - Check(PatchSrtReads(program, &error) && TrackResources(program, &error), error.c_str()); + ShaderComputeInputInfo compute{}; + Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && + AllocateBindings(fixture.program, {}, &error) && + FindBinding(fixture.program.bindings, + DescriptorBindingKind::FlattenedSrt) != nullptr, + "flattened typed SRT reads did not receive a binding"); } void TestDynamicSrtReadRemainsExplicit() { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 8; i++) { - auto load = ScalarLoad(i * 4, i, 16, i * 4); - load.src[0] = Sgpr(20); - load.src_count = 1; - insts.push_back(load); - } - insts.push_back(ImageUse(0x40, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error), - error.c_str()); - Check(program.srt.reads.empty() && program.srt.dynamic_reads.size() == 8 && - program.srt_patching_complete, - "dynamic SRT offsets were incorrectly assigned fixed flat slots"); - for (uint32_t i = 0; i < 8; i++) { - Check(insts[i].op == Opcode::SLoadDword && - insts[i].memory.kind == ResourceKind::ScalarBuffer, - "dynamic SRT read was rewritten as an immediate flat load"); - } - Check(TrackResources(program, &error), error.c_str()); + Fixture fixture; + const auto base = + fixture.Address(fixture.UserData(0), fixture.UserData(1), 4); + MemoryInfo scalar; + scalar.kind = ResourceKind::ScalarBuffer; + const auto read = + fixture.Emit(ValueOpcode::LoadAddressU32, + {base, fixture.UserData(2), Value(0u), Value(true)}, + fixture.AddMemory(scalar, 4)); + const auto descriptor = + fixture.Buffer({read, Value(0u), Value(64u), Value(0u)}, 8); + MemoryInfo buffer; + buffer.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {descriptor, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(buffer, 8)); + fixture.PlanAndTrack(); + + Check(fixture.program.values->srt_reads.empty() && + fixture.program.values->dynamic_reads.size() == 1 && + fixture.program.info.addresses.size() == 1, + "dynamic scalar read was incorrectly flattened or lost"); + std::array user_data{0x1000u, 0u, 4u}; + TestMemory memory; + memory.words[1] = 0xabcdef01u; + SrtRuntime runtime{.user_data = user_data, + .read_memory = ReadTestMemory, + .userdata = &memory}; + DescriptorValue value; + std::string error; + Check(EvaluateDescriptorSource(fixture.program, + fixture.program.info.buffers[0].source, 8, + runtime, value, &error) && + value.dwords[0] == 0xabcdef01u && memory.reads == 1, + "dynamic typed scalar descriptor source was not evaluated"); + + ShaderComputeInputInfo compute{}; + Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error) && + AllocateBindings(fixture.program, {}, &error) && + FindBinding(fixture.program.bindings, + DescriptorBindingKind::FlattenedSrt) == nullptr && + FindBinding(fixture.program.bindings, + DescriptorBindingKind::AddressMemory) != nullptr, + "dynamic scalar read received the wrong resource bindings"); } -void TestSrtPatchingFailureIsTransactional() { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); - } - insts.push_back(BufferUse(0x20, 0)); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - program.srt.reads.size() == 4, - error.c_str()); - insts[0].scalar_value = ScalarProvenance::Undefined; - const auto instructions = insts; - const auto provenance = program.provenance; - const auto srt = program.srt; - Check(!PatchSrtReads(program, &error) && - error.find("no scalar-load producer") != std::string::npos && insts == instructions && - program.provenance == provenance && program.srt == srt && - !program.srt_patching_complete, - "failed SRT patching partially changed the program"); +void TestPhiValidation() { + Fixture fixture; + auto *left = fixture.block; + auto *right = fixture.AddBlock(); + auto *merge = fixture.AddBlock(); + left->AddBranch(merge); + right->AddBranch(merge); + auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {}, + static_cast(Type::U32)); + phi.AddPhiOperand(left, Value(1u)); + phi.AddPhiOperand(right, Value(2u)); + const auto handle = + fixture.Emit(ValueOpcode::GetBufferResource, + {Value(&phi), Value(0u), Value(0u), Value(0u)}, + MemoryFlags{0, 20}, merge); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {handle, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(memory, 20), merge); + + std::string error; + Check(BuildSrtPlan(fixture.program, &error), + "SRT planning rejected a well-formed typed phi"); + Check(!TrackResources(fixture.program, &error) && + error.find("control-dependent phi") != std::string::npos && + !fixture.program.resource_tracking_complete && + fixture.program.info.buffers.empty() && + fixture.program.values->descriptor_sources.empty(), + "control-dependent descriptor phi was not rejected transactionally"); } -void TestScalarMemoryGroupsSnapshotOperands() { - const auto CheckGroup = [](bool buffer) { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(buffer ? ScalarBufferLoad(4, 16 + i, 4, i * 4) - : ScalarLoad(4, 16 + i, 16, i * 4)); - } - MakeScalarMemoryGroup(&insts); - insts.push_back(BufferUse(8, 16)); +void TestInvariantLoopPhi() { + Fixture fixture; + auto *entry = fixture.block; + auto *loop = fixture.AddBlock(); + entry->AddBranch(loop); + loop->AddBranch(loop); + const auto invariant = fixture.UserData(0); + auto &phi = loop->AppendNewInst(ValueOpcode::Phi, {}, + static_cast(Type::U32)); + phi.AddPhiOperand(entry, invariant); + phi.AddPhiOperand(loop, Value(&phi)); + const auto handle = fixture.Emit( + ValueOpcode::GetBufferResource, + {Value(&phi), Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 4}, loop); + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {handle, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(memory, 4), loop); + fixture.PlanAndTrack(); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - program.srt.reads.size() == 4 && PatchSrtReads(program, &error), - error.c_str()); - for (uint32_t i = 0; i < 4; i++) { - Check(insts[i].op == Opcode::LoadSrtDword && insts[i].src[0].imm == i, - "overlapping scalar-memory group did not snapshot its operands"); - } - }; - CheckGroup(false); - CheckGroup(true); - - const auto CheckOffsetOverlap = [](bool buffer) { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 4; i++) { - auto load = - buffer ? ScalarBufferLoad(4, 20 + i, 4, i * 4) : ScalarLoad(4, 20 + i, 16, i * 4); - load.src[0] = Sgpr(20); - load.src_count = 1; - insts.push_back(load); - } - MakeScalarMemoryGroup(&insts); - insts.push_back(BufferUse(8, 20)); - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - program.srt.dynamic_reads.size() == 4 && PatchSrtReads(program, &error), - error.c_str()); - for (uint32_t i = 0; i < 4; i++) { - const auto value = insts[i].scalar_value; - const auto& node = program.provenance.values[value]; - const auto offset_arg = buffer ? 4u : 2u; - Check(program.provenance.values[node.args[offset_arg]].op == ScalarValueOp::UserData && - program.provenance.values[node.args[offset_arg]].imm == 20 && - insts[i].op == (buffer ? Opcode::SBufferLoadDword : Opcode::SLoadDword), - "overlapping scalar-memory offset was read after a component write"); - } - }; - CheckOffsetOverlap(false); - CheckOffsetOverlap(true); + std::array user_data{0x12345678u}; + SrtRuntime runtime{.user_data = user_data}; + DescriptorValue descriptor; + std::string error; + Check(EvaluateDescriptorSource(fixture.program, + fixture.program.info.buffers[0].source, 4, + runtime, descriptor, &error) && + descriptor.dwords[0] == user_data[0], + "loop-invariant descriptor phi was not evaluated through typed SSA"); } -void TestSrtPatchingHandlesGvnAndMoveForwarding() { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t copy = 0; copy < 2; copy++) { - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(ScalarLoad(copy * 0x20 + i * 4, copy * 4 + i, 16, i * 4)); - } - } - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(Move(0x40 + i * 4, 8 + i, i)); - } - insts.push_back(BufferUse(0x60, 4)); - insts.push_back(BufferUse(0x64, 8)); +void TestAddressMaterializationAndSpecialization() { + Fixture fixture; + const auto based = + fixture.Address(fixture.UserData(0), fixture.UserData(1), 4); + MemoryInfo global; + global.kind = ResourceKind::Global; + global.offset = static_cast(-8); + fixture.Emit(ValueOpcode::LoadAddressU32, + {based, Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(global, 4)); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - program.srt.reads.size() == 4 && PatchSrtReads(program, &error), - error.c_str()); - for (uint32_t i = 0; i < 8; i++) { - Check(insts[i].op == Opcode::LoadSrtDword && insts[i].src[0].imm == i % 4, - "all producers of a GVN'd SRT read were not patched"); - } - for (uint32_t i = 8; i < 12; i++) { - Check(insts[i].op == Opcode::MoveU32, - "move forwarding was incorrectly rewritten as an SRT load"); - } + const auto undef = fixture.Emit(ValueOpcode::UndefU32); + const auto unbased = fixture.Address(undef, undef, 8); + MemoryInfo flat; + flat.kind = ResourceKind::Flat; + flat.address_is_full = true; + fixture.Emit(ValueOpcode::StoreAddressU32, + {unbased, Value(0u), Value(0u), Value(9u), Value(true)}, + fixture.AddMemory(flat, 8)); + fixture.PlanAndTrack(); + + Check(fixture.program.info.addresses.size() == 2 && + !fixture.program.info.addresses[0].unbased && + fixture.program.info.addresses[0].min_offset == -8 && + fixture.program.info.addresses[1].unbased, + "typed based and unbased addresses were classified incorrectly"); + std::array user_data{0x2008u, 0u}; + SrtRuntime runtime{.user_data = user_data, .flat_memory_base = 0x9000u}; + ResourceSnapshot snapshot; + std::string error; + Check(MaterializeResources(fixture.program, runtime, snapshot, &error), + "address resources did not materialize"); + Check(snapshot.addresses.size() == 2 && + snapshot.addresses[0].guest_base == 0x2008u && + snapshot.addresses[0].binding_base == 0x2000u && + snapshot.addresses[1].binding_base == 0x9000u, + "materialized address windows are incorrect"); + Check(SpecializeResources(fixture.program, snapshot, &error) && + fixture.program.info.addresses[0].specialized_base == 8u && + fixture.program.info.addresses[1].specialized_base == 0x9000u, + "typed address specialization was not applied"); } -void TestSrtPatchingHandlesCfgProducers() { - Program program; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - for (uint32_t block = 1; block <= 2; block++) { - for (uint32_t i = 0; i < 4; i++) { - program.blocks[block].instructions.push_back( - ScalarLoad(block * 0x20 + i * 4, i, 16, i * 4)); - } - } - program.blocks[3].instructions = {BufferUse(0x60, 0)}; +void TestShaderInfoAndBindingLayout() { + Fixture fixture; + const auto handle = fixture.Buffer( + {fixture.UserData(3), fixture.UserData(4), Value(64u), Value(0u)}, 4); + MemoryInfo buffer; + buffer.kind = ResourceKind::Buffer; + fixture.Emit(ValueOpcode::LoadBufferU32, + {handle, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(buffer, 4)); + fixture.Emit( + ValueOpcode::GetBuiltin, + {Value(static_cast(StageInputKind::GlobalInvocationId)), + Value(2u)}); + fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)}); + const auto gds = fixture.Emit(ValueOpcode::GetGdsResource); + fixture.Emit(ValueOpcode::WriteGdsU32, + {gds, Value(0u), Value(1u), Value(true)}); + fixture.PlanAndTrack(); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - program.srt.reads.size() == 4 && PatchSrtReads(program, &error), - error.c_str()); - for (uint32_t block = 1; block <= 2; block++) { - for (uint32_t i = 0; i < 4; i++) { - const auto& inst = program.blocks[block].instructions[i]; - Check(inst.op == Opcode::LoadSrtDword && inst.src[0].imm == i, - "CFG-equivalent SRT producer was not patched"); - } - } + ShaderComputeInputInfo compute{}; + compute.dispatch_thread_dimensions = true; + std::string error; + Check(CollectShaderInfo(fixture.program, {.compute = &compute}, &error), + "typed shader info collection failed"); + Check(fixture.program.info.has_bitwise_xor && + !fixture.program.info.inputs.empty() && + fixture.program.info.inputs[0].kind == + StageInputKind::GlobalInvocationId, + "typed shader values were not reflected in shader info"); + + BindingLayoutOptions options; + options.descriptor_set = 2; + options.max_push_dwords = 1; + Check(AllocateBindings(fixture.program, options, &error), + "typed binding allocation failed"); + Check(fixture.program.bindings.descriptor_set == 2 && + FindBinding(fixture.program.bindings, + DescriptorBindingKind::Buffers) != nullptr && + FindBinding(fixture.program.bindings, DescriptorBindingKind::Gds) != + nullptr && + FindBinding(fixture.program.bindings, + DescriptorBindingKind::UserData) != nullptr, + "typed resources were not assigned native bindings"); + Check(fixture.program.bindings.user_data_registers == + std::vector({3u, 4u}), + "binding layout did not collect live typed user-data values"); } -void TestSrtPatchPlanValidation() { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); - } - insts.push_back(BufferUse(0x20, 0)); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - program.srt.reads = {program.srt.reads[0], program.srt.reads[0]}; - const auto instructions = insts; - Check(!PatchSrtReads(program, &error) && - error.find("dense value-to-offset bijection") != std::string::npos && - insts == instructions && !program.srt_patching_complete, - "duplicate SRT flat slots were accepted or partially patched"); - - Program dynamic; - dynamic.blocks.resize(1); - dynamic.blocks[0].instructions = {BufferUse(4, 0)}; - Check(BuildScalarProvenance(dynamic, &error) && BuildSrtPlan(dynamic, &error), error.c_str()); - dynamic.srt.dynamic_sources = {ScalarProvenance::Undefined}; - Check(!PatchSrtReads(dynamic, &error) && - error.find("invalid dynamic descriptor source") != std::string::npos && - !dynamic.srt_patching_complete, - "undefined dynamic descriptor source was accepted"); -} - -void TestMaterializationSharesReadConstEvaluation() { - Program program; - program.stage = ShaderType::Pixel; - program.shader_hash = 0x10203040; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 8; i++) { - insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); - } - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(MoveImmediate(0x20 + i * 4, 8 + i, 0xa0 + i)); - } - insts.push_back( - ImageUse(0x40, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D)); - insts.push_back(ImageUse(0x44, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)); - Prepare(program); - Check(program.info.images.size() == 2 && program.info.samplers.size() == 1, - "materialization test did not preserve sampled/storage view topology"); - Check(program.srt.reads.size() == 8 && program.srt_patching_complete, - "immediate descriptor reads did not produce a compact SRT patch plan"); - for (uint32_t i = 0; i < 8; i++) { - Check(insts[i].op == Opcode::LoadSrtDword && insts[i].src_count == 1 && - insts[i].src[0].kind == OperandKind::ImmediateU32 && insts[i].src[0].imm == i && - insts[i].memory.kind == ResourceKind::None, - "immediate SRT read was not patched to its dense flat-buffer slot"); - } - std::string error; - Check(!PatchSrtReads(program, &error) && error.find("already patched") != std::string::npos, - "repeated SRT patching was accepted"); - - TestMemory memory; - for (uint32_t i = 0; i < memory.words.size(); i++) { - memory.words[i] = 0x100 + i; - } - memory.words[1] |= static_cast(Prospero::BufferFormat::k8UNorm) << 20u; - memory.words[3] |= static_cast(Prospero::ImageType::kColor2D) << 28u; - std::array user_data {}; - user_data[16] = static_cast(memory.base); - user_data[17] = static_cast(memory.base >> 32u); - SrtRuntime runtime {user_data, 0, ReadTestMemory, &memory}; - ResourceSnapshot snapshot; - error.clear(); - Check(MaterializeResources(program, runtime, snapshot, &error), error.c_str()); - Check(snapshot.buffers.empty() && snapshot.images.size() == 2 && - snapshot.samplers.size() == 1 && snapshot.images[0] == snapshot.images[1], - "dense runtime snapshot did not preserve resource order or aliases"); - Check(snapshot.flattened_srt.size() == 8 && - std::equal(snapshot.flattened_srt.begin(), snapshot.flattened_srt.end(), - memory.words.begin()) && - snapshot.user_data == std::vector(user_data.begin(), user_data.end()), - "runtime snapshot omitted flattened SRT or current user data"); - Check(memory.reads == 8, "aliased image views repeated ReadConst evaluation " - "instead of sharing it"); - for (uint32_t i = 0; i < 8; i++) { - Check(snapshot.images[0].dwords[i] == memory.words[i], - "materialized image descriptor contains the wrong dword"); - } -} - -void TestInvalidImagesMaterializeAsNull() { - Program sampled; - sampled.stage = ShaderType::Pixel; - sampled.user_data_count = 8; - sampled.blocks.resize(1); - sampled.blocks[0].instructions = { - ImageUse(0x40, Opcode::ImageLoad, ResourceKind::Image, Decoder::ImageDimension::Dim2D)}; - Prepare(sampled); - Check(sampled.info.images.size() == 1 && sampled.info.samplers.empty(), - "sampled-image normalization test has unexpected resource topology"); - - constexpr std::array stale = {0x00000004, 0x00000004, 0xc0061060, 0x06000514, - 0x20010000, 0xa4580290, 0x00000004, 0x00000001}; - std::string error; - for (uint32_t type = 0; type < 8; type++) { - auto descriptor = stale; - descriptor[1] |= static_cast(Prospero::BufferFormat::k8UNorm) << 20u; - descriptor[3] = (descriptor[3] & 0x0fffffffu) | (type << 28u); - ResourceSnapshot snapshot; - Check(MaterializeResources(sampled, {descriptor}, snapshot, &error) && - snapshot.images.size() == 1 && - std::all_of(snapshot.images[0].dwords.begin(), snapshot.images[0].dwords.end(), - [](uint32_t word) { return word == 0; }) && - ValidateResourceSpecialization(sampled, snapshot, &error), - error.c_str()); - } - - constexpr std::array packet = {0xc0071058, 0xe80eeeb8, 0x00000000, 0xffffffff, - 0xffffffff, 0x00000001, 0x00000000, 0x00000113}; - ResourceSnapshot packet_snapshot; - Check(MaterializeResources(sampled, {packet}, packet_snapshot, &error) && - std::all_of(packet_snapshot.images[0].dwords.begin(), - packet_snapshot.images[0].dwords.end(), - [](uint32_t word) { return word == 0; }), - "invalid MSAA image words were not normalized to null"); - - struct MsaaCase { - uint32_t base_level; - uint32_t fragments; - uint32_t max_mip; - bool valid; - }; - constexpr std::array msaa_cases = { - MsaaCase {0, 1, 1, true}, MsaaCase {0, 2, 2, true}, MsaaCase {0, 3, 3, true}, - MsaaCase {1, 1, 1, false}, MsaaCase {0, 2, 1, false}, MsaaCase {0, 0, 0, false}, - MsaaCase {0, 4, 4, false}, - }; - constexpr std::array msaa_types = { - Prospero::ImageType::kColor2DMsaa, - Prospero::ImageType::kColor2DMsaaArray, - }; - for (const auto type: msaa_types) { - for (const auto& test: msaa_cases) { - std::array msaa {}; - msaa[0] = 1; - msaa[1] = 36u << 20u; - msaa[3] = (static_cast(type) << 28u) | (test.base_level << 12u) | - (test.fragments << 16u); - msaa[5] = test.max_mip << 4u; - ResourceSnapshot msaa_snapshot; - const auto materialized = MaterializeResources(sampled, {msaa}, msaa_snapshot, &error); - const auto preserved = - materialized && - std::equal(msaa.begin(), msaa.end(), msaa_snapshot.images[0].dwords.begin()); - const auto is_null = - materialized && std::all_of(msaa_snapshot.images[0].dwords.begin(), - msaa_snapshot.images[0].dwords.end(), - [](uint32_t word) { return word == 0; }); - Check(materialized && (test.valid ? preserved : is_null), - "MSAA image descriptor validity mismatch"); - } - } - - std::array invalid_format {}; - invalid_format[0] = 1; - invalid_format[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; - ResourceSnapshot invalid_format_snapshot; - Check(MaterializeResources(sampled, {invalid_format}, invalid_format_snapshot, &error) && - std::all_of(invalid_format_snapshot.images[0].dwords.begin(), - invalid_format_snapshot.images[0].dwords.end(), - [](uint32_t word) { return word == 0; }), - "invalid-format image descriptor was not normalized to null"); - - auto valid = stale; - valid[1] |= static_cast(Prospero::BufferFormat::k8UNorm) << 20u; - valid[3] = - (valid[3] & 0x0fffffffu) | (static_cast(Prospero::ImageType::kColor2D) << 28u); - ResourceSnapshot valid_snapshot; - Check(MaterializeResources(sampled, {valid}, valid_snapshot, &error) && - std::equal(valid.begin(), valid.end(), valid_snapshot.images[0].dwords.begin()), - "valid sampled image descriptor was normalized"); - - Program storage; - storage.stage = ShaderType::Compute; - storage.user_data_count = 8; - storage.blocks.resize(1); - storage.blocks[0].instructions = {ImageUse(0x40, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)}; - Prepare(storage); - ResourceSnapshot storage_snapshot; - Check(MaterializeResources(storage, {stale}, storage_snapshot, &error) && - std::all_of(storage_snapshot.images[0].dwords.begin(), - storage_snapshot.images[0].dwords.end(), - [](uint32_t word) { return word == 0; }), - "invalid storage image descriptor was not normalized to null"); -} - -void TestInvalidBuffersMaterializeAsNull() { - Program program; - program.stage = ShaderType::Compute; - program.user_data_count = 4; - program.blocks.resize(1); - program.blocks[0].instructions = {BufferUse(0x40, 0)}; - Prepare(program); - - constexpr std::array valid = {0x0000100c, 0x00100000, 0x00000001, 0x0004dfac}; - auto invalid = valid; - invalid[3] |= 1u << 30u; - std::string error; - ResourceSnapshot snapshot; - Check(MaterializeResources(program, {invalid}, snapshot, &error) && - std::all_of(snapshot.buffers[0].dwords.begin(), - snapshot.buffers[0].dwords.begin() + 4, - [](uint32_t word) { return word == 0; }), - "invalid buffer descriptor was not normalized to null"); - Check(MaterializeResources(program, {valid}, snapshot, &error) && - std::equal(valid.begin(), valid.end(), snapshot.buffers[0].dwords.begin()), - "valid buffer descriptor was normalized"); -} - -void TestMaterializationFailureIsTransactional() { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 8; i++) { - insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); - } - insts.push_back(ImageUse(0x40, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)); - Prepare(program); - - TestMemory memory; - memory.fail_after = 3; - std::array user_data {}; - user_data[16] = static_cast(memory.base); - user_data[17] = static_cast(memory.base >> 32u); - SrtRuntime runtime {user_data, 0, ReadTestMemory, &memory}; - ResourceSnapshot snapshot; - DescriptorValue sentinel; - sentinel.dword_count = 1; - sentinel.dwords[0] = 777; - snapshot.images.push_back(sentinel); - std::string error; - Check(!MaterializeResources(program, runtime, snapshot, &error) && - error.find("failed at") != std::string::npos && snapshot.images.size() == 1 && - snapshot.images[0].dwords[0] == 777 && snapshot.images[0].dword_count == 1, - "failed runtime materialization partially replaced the prior snapshot"); -} - -void TestResourceSpecializationIsTypedAndTransactional() { - Program null_program; - null_program.stage = ShaderType::Compute; - null_program.blocks.resize(1); - null_program.blocks[0].instructions = { - ImageUse(0x10, Opcode::ImageLoad, ResourceKind::ImageUint, Decoder::ImageDimension::Dim3D)}; - Prepare(null_program); - ResourceSnapshot null_snapshot; - null_snapshot.images.resize(1); - null_snapshot.images[0].dword_count = 8; - null_snapshot.images[0].dwords[1] = 0x12345600u; - null_snapshot.images[0].dwords[2] = 0x89abcdefu; - null_snapshot.images[0].dwords[3] = 0x01234567u; - std::string error; - Check(SpecializeResources(null_program, null_snapshot, &error) && - ValidateResourceSpecialization(null_program, null_snapshot, &error) && - null_program.info.images[0].kind == ResourceKind::Image && - null_program.info.images[0].dimension == Decoder::ImageDimension::Dim2D && - null_program.blocks[0].instructions[0].memory.image_dimension == - Decoder::ImageDimension::Dim2D, - "zero-base image descriptor did not use the canonical null-image shape"); - - Program null_atomic; - null_atomic.stage = ShaderType::Compute; - null_atomic.blocks.resize(1); - null_atomic.blocks[0].instructions = {ImageUse(0x14, Opcode::AtomicAddU32, - ResourceKind::StorageImageUint, - Decoder::ImageDimension::Dim1D)}; - Prepare(null_atomic); - Check(SpecializeResources(null_atomic, null_snapshot, &error) && - ValidateResourceSpecialization(null_atomic, null_snapshot, &error) && - null_atomic.info.images[0].kind == ResourceKind::StorageImageUint && - null_atomic.info.images[0].dimension == Decoder::ImageDimension::Dim2D, - "null image atomic did not preserve its required integer storage type"); - - Program array_view; - array_view.stage = ShaderType::Compute; - array_view.blocks.resize(1); - array_view.blocks[0].instructions = { - ImageUse(0x18, Opcode::ImageLoad, ResourceKind::Image, Decoder::ImageDimension::Dim1D)}; - Prepare(array_view); - ResourceSnapshot array_snapshot; - array_snapshot.images.resize(1); - array_snapshot.images[0].dword_count = 8; - array_snapshot.images[0].dwords[0] = 0x1000; - array_snapshot.images[0].dwords[3] = static_cast(Prospero::ImageType::kColor1DArray) - << 28u; - Check(SpecializeResources(array_view, array_snapshot, &error) && - array_view.info.images[0].dimension == Decoder::ImageDimension::Dim1D && - array_view.blocks[0].instructions[0].memory.image_dimension == - Decoder::ImageDimension::Dim1D, - "non-array MIMG view did not narrow a 1D-array descriptor"); - auto null_after_1d = array_snapshot; - null_after_1d.images[0].dwords.fill(0); - Check(!ValidateResourceSpecialization(array_view, null_after_1d, &error) && - error.find("canonical null specialization") != std::string::npos, - "non-null 1D specialization accepted a canonical 2D null descriptor"); - - Program cross_family_array; - cross_family_array.stage = ShaderType::Compute; - cross_family_array.blocks.resize(1); - cross_family_array.blocks[0].instructions = {ImageUse( - 0x1c, Opcode::ImageLoad, ResourceKind::Image, Decoder::ImageDimension::Dim2DArray)}; - Prepare(cross_family_array); - Check(SpecializeResources(cross_family_array, array_snapshot, &error) && - cross_family_array.info.images[0].dimension == Decoder::ImageDimension::Dim1DArray && - cross_family_array.blocks[0].instructions[0].memory.image_dimension == - Decoder::ImageDimension::Dim1DArray, - "array MIMG intent did not produce a 1D-array view"); - - Program cross_family_2d_array; - cross_family_2d_array.stage = ShaderType::Compute; - cross_family_2d_array.blocks.resize(1); - cross_family_2d_array.blocks[0].instructions = {ImageUse( - 0x20, Opcode::ImageLoad, ResourceKind::Image, Decoder::ImageDimension::Dim1DArray)}; - Prepare(cross_family_2d_array); - auto array_2d_snapshot = array_snapshot; - array_2d_snapshot.images[0].dwords[3] = - static_cast(Prospero::ImageType::kColor2DArray) << 28u; - Check(SpecializeResources(cross_family_2d_array, array_2d_snapshot, &error) && - cross_family_2d_array.info.images[0].dimension == Decoder::ImageDimension::Dim2DArray, - "array MIMG intent did not produce a 2D-array view"); - - Program cube_view; - cube_view.stage = ShaderType::Compute; - cube_view.blocks.resize(1); - cube_view.blocks[0].instructions = {ImageUse(0x24, Opcode::ImageLoad, ResourceKind::Image, - Decoder::ImageDimension::Dim2DArray)}; - Prepare(cube_view); - auto cube_snapshot = array_2d_snapshot; - cube_snapshot.images[0].dwords[3] = static_cast(Prospero::ImageType::kCube) << 28u; - Check(SpecializeResources(cube_view, cube_snapshot, &error) && - ValidateResourceSpecialization(cube_view, cube_snapshot, &error) && - cube_view.info.images[0].cube && - cube_view.blocks[0].instructions[0].memory.image_cube, - "cube descriptor identity did not reach the specialized image and IR"); - auto array_after_cube = cube_snapshot; - array_after_cube.images[0].dwords[3] = - static_cast(Prospero::ImageType::kColor2DArray) << 28u; - Check(!ValidateResourceSpecialization(cube_view, array_after_cube, &error), - "2D-array descriptor reused a cube-coordinate specialization"); - auto null_after_cube = cube_snapshot; - null_after_cube.images[0].dwords.fill(0); - Check(SpecializeResources(cube_view, null_after_cube, &error) && - ValidateResourceSpecialization(cube_view, null_after_cube, &error) && - !cube_view.info.images[0].cube && - !cube_view.blocks[0].instructions[0].memory.image_cube, - "canonical null respecialization retained stale cube-coordinate state"); - - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - program.blocks[0].instructions = {ImageUse(0x20, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim3D)}; - Prepare(program); - - ResourceSnapshot snapshot; - snapshot.images.resize(1); - snapshot.images[0].dword_count = 7; - const auto info = program.info; - const auto memory = program.blocks[0].instructions[0].memory; - error.clear(); - Check(!ValidateResourceSnapshot(program, snapshot, &error) && - error.find("image descriptor 0 has 7 dwords") != std::string::npos, - "malformed resource snapshot was accepted"); - Check(!SpecializeResources(program, snapshot, &error) && program.info == info && - program.blocks[0].instructions[0].memory == memory, - "failed resource specialization partially mutated the program"); - - snapshot.images[0].dword_count = 8; - snapshot.images[0].dwords[0] = 0x1000; - snapshot.images[0].dwords[1] = static_cast(Prospero::BufferFormat::k32UInt) << 20u; - snapshot.images[0].dwords[3] = - (static_cast(Prospero::ImageType::kColor2D) << 28u) | 0x3acu; - Check(SpecializeResources(program, snapshot, &error), error.c_str()); - const auto& image = program.info.images[0]; - const auto& inst = program.blocks[0].instructions[0]; - Check(image.kind == ResourceKind::StorageImageUint && - image.dimension == Decoder::ImageDimension::Dim2D && - image.storage_swizzle == 0x3acu && - inst.memory.kind == ResourceKind::StorageImageUint && - inst.memory.image_dimension == Decoder::ImageDimension::Dim2D, - "runtime descriptor shape and integer format did not specialize dense " - "IR"); - auto null_after_uint = snapshot; - null_after_uint.images[0].dwords.fill(0); - Check(!ValidateResourceSpecialization(program, null_after_uint, &error) && - error.find("canonical null specialization") != std::string::npos, - "non-null integer specialization accepted a non-integer null descriptor"); - - auto stale_swizzle = snapshot; - stale_swizzle.images[0].dwords[3] = - (stale_swizzle.images[0].dwords[3] & ~0xfffu) | StorageImageIdentitySwizzle; - Check(!ValidateResourceSpecialization(program, stale_swizzle, &error) && - error.find("changed swizzle") != std::string::npos, - "storage image cache validation ignored a SPIR-V-baked swizzle"); - - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - Check(CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - const auto* binding = FindBinding(program.bindings, DescriptorBindingKind::StorageUint2D); - Check(binding != nullptr && binding->resources == std::vector({0}) && - FindBinding(program.bindings, DescriptorBindingKind::Storage3D) == nullptr, - "specialized image topology did not reach the exact native binding " - "group"); -} - -void TestRuntimeSpecializationCoversBakedBufferAndAddressFields() { - Program buffer_program; - buffer_program.stage = ShaderType::Compute; - buffer_program.blocks.resize(1); - buffer_program.blocks[0].instructions = {BufferUse(0, 0)}; - Prepare(buffer_program); - - ResourceSnapshot buffer_snapshot; - buffer_snapshot.buffers.resize(1); - buffer_snapshot.buffers[0].dword_count = 4; - buffer_snapshot.buffers[0].dwords[1] = (16u << 16u) | (1u << 31u); - buffer_snapshot.buffers[0].dwords[3] = - (static_cast(Prospero::BufferFormat::k32UInt) << 12u) | (2u << 21u) | - (1u << 23u); - std::string error; - Check(SpecializeResources(buffer_program, buffer_snapshot, &error) && - ValidateResourceSpecialization(buffer_program, buffer_snapshot, &error), - error.c_str()); - Check(buffer_program.info.buffers[0].packed_stride == - (16u | (1u << 14u) | (2u << 16u) | (1u << 20u)) && - buffer_program.info.buffers[0].descriptor_format == - Prospero::BufferFormat::k32UInt, - "buffer specialization omitted SPIR-V-baked descriptor fields"); - auto stale_buffer = buffer_snapshot; - stale_buffer.buffers[0].dwords[1] ^= 4u << 16u; - Check(!ValidateResourceSpecialization(buffer_program, stale_buffer, &error) && - error.find("buffer descriptor 0") != std::string::npos, - "stale buffer stride reused incompatible specialized SPIR-V"); - - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - Check(CollectShaderInfo(buffer_program, {.compute = &compute}, &error) && - AllocateBindings(buffer_program, {}, &error), - error.c_str()); - Check(buffer_program.bindings.user_data_registers == std::vector({0, 1, 2, 3}), - "buffer fixture did not allocate its descriptor user-data roots"); - buffer_snapshot.user_data.resize(4); - Check(ValidateResourceSpecialization(buffer_program, buffer_snapshot, &error), error.c_str()); - auto truncated_user_data = buffer_snapshot; - truncated_user_data.user_data.resize(3); - Check(!ValidateResourceSnapshot(buffer_program, truncated_user_data, &error) && - error.find("user SGPR 3") != std::string::npos, - "truncated cache snapshot user-data window was accepted"); - - Program based_program; - based_program.stage = ShaderType::Compute; - based_program.blocks.resize(1); - auto raw = ScalarLoad(8, 0, 16, 0); - raw.memory.offset = static_cast(-4); - raw.src[0] = Sgpr(20); - raw.src_count = 1; - based_program.blocks[0].instructions = {MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), - raw}; - Prepare(based_program); - ResourceSnapshot based_snapshot; - Check(MaterializeResources(based_program, {}, based_snapshot, &error) && - SpecializeResources(based_program, based_snapshot, &error), - error.c_str()); - auto relocated = based_snapshot; - relocated.addresses[0].guest_base += 0x1000; - relocated.addresses[0].binding_base += 0x1000; - Check(ValidateResourceSpecialization(based_program, relocated, &error), - "relocated based address with identical relative bias changed " - "specialization"); - relocated.addresses[0].binding_base++; - Check(!ValidateResourceSpecialization(based_program, relocated, &error) && - error.find("address resource 0") != std::string::npos, - "stale based-address bias reused incompatible specialized SPIR-V"); - - Program flat_program; - flat_program.stage = ShaderType::Compute; - flat_program.blocks.resize(1); - Instruction flat; - flat.op = Opcode::FlatLoadDword; - flat.memory.kind = ResourceKind::Flat; - flat_program.blocks[0].instructions = {flat}; - Prepare(flat_program); - ResourceSnapshot flat_snapshot; - flat_snapshot.addresses = {{0x11u, 0x10u}}; - flat_snapshot.user_data = {0xdeadbeefu}; - const auto prior_flat_snapshot = flat_snapshot; - Check(!MaterializeResources(flat_program, {}, flat_snapshot, &error) && - error.find("requires runtime guest-address translation") != std::string::npos && - flat_snapshot.addresses == prior_flat_snapshot.addresses && - flat_snapshot.user_data == prior_flat_snapshot.user_data, - "unbased flat memory without a translator did not fail transactionally"); - SrtRuntime flat_runtime; - flat_runtime.flat_memory_base = 0x100000000ull; - Check(MaterializeResources(flat_program, flat_runtime, flat_snapshot, &error) && - SpecializeResources(flat_program, flat_snapshot, &error), - error.c_str()); - auto stale_flat = flat_snapshot; - stale_flat.addresses[0].guest_base += 0x1000; - stale_flat.addresses[0].binding_base += 0x1000; - Check(!ValidateResourceSpecialization(flat_program, stale_flat, &error), - "unbased flat memory reused an absolute base baked into SPIR-V"); -} - -void TestTrackedProgramIsImmutable() { - Program program; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - for (uint32_t i = 0; i < 8; i++) { - insts.push_back(ScalarLoad(i * 4, i, 16, i * 4)); - } - for (uint32_t i = 0; i < 4; i++) { - insts.push_back(MoveImmediate(0x20 + i * 4, 8 + i, 0xa0 + i)); - insts.push_back(MoveImmediate(0x30 + i * 4, 12 + i, 0xb0 + i)); - } - insts.push_back( - ImageUse(0x50, Opcode::ImageSample, ResourceKind::Image, Decoder::ImageDimension::Dim2D)); - insts.push_back(ImageUse(0x54, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)); - insts.push_back(BufferUse(0x58, 12)); - Prepare(program); - Check(program.info.buffers.size() == 1 && program.info.images.size() == 2 && - program.info.samplers.size() == 1 && program.info.sampled_pairs.size() == 1, - "post-track immutability fixture lacks complete resource topology"); - - const auto provenance = program.provenance; - const auto srt = program.srt; - const auto info = program.info; - const auto srt_complete = program.srt_plan_complete; - const auto tracking_complete = program.resource_tracking_complete; - const auto patching_complete = program.srt_patching_complete; - std::vector memory; - memory.reserve(insts.size()); - for (const auto& inst: insts) { - memory.push_back(inst.memory); - } - - const auto CheckUnchanged = [&]() { - Check(program.provenance == provenance && program.srt == srt && program.info == info && - program.srt_plan_complete == srt_complete && - program.srt_patching_complete == patching_complete && - program.resource_tracking_complete == tracking_complete && - insts.size() == memory.size(), - "rejected post-track pass mutated immutable program state"); - for (uint32_t i = 0; i < memory.size(); i++) { - Check(insts[i].memory == memory[i], - "rejected post-track pass mutated a dense operand or source handle"); - } - }; - - std::string error; - Check(!BuildScalarProvenance(program, &error) && - error.find("after resource tracking") != std::string::npos, - "post-track provenance rebuild was accepted"); - CheckUnchanged(); - Check(!BuildSrtPlan(program, &error) && - error.find("after resource tracking") != std::string::npos, - "post-track SRT rebuild was accepted"); - CheckUnchanged(); -} - -void TestNativeBindingLayout() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - insts.push_back(BufferUse(4, 48)); - insts.push_back(ImageUse(8, Opcode::ImageSample, ResourceKind::Image, - Decoder::ImageDimension::Dim2D, 0, 2)); - insts.push_back(ImageUse(12, Opcode::ImageSample, ResourceKind::Image, - Decoder::ImageDimension::Dim2DArray, 0, 2)); - insts.push_back(ImageUse(16, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim3D, 4)); - insts.push_back(ImageUse(20, Opcode::ImageStore, ResourceKind::StorageImageUint, - Decoder::ImageDimension::Dim2DArray, 6)); - Prepare(program); - - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - std::string error; - Check(CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {.descriptor_set = 3}, &error), - error.c_str()); - const auto* buffers = FindBinding(program.bindings, DescriptorBindingKind::Buffers); - const auto* sampled2d = FindBinding(program.bindings, DescriptorBindingKind::Sampled2D); - const auto* sampled_array = - FindBinding(program.bindings, DescriptorBindingKind::Sampled2DArray); - const auto* storage3d = FindBinding(program.bindings, DescriptorBindingKind::Storage3D); - const auto* storage_uint_array = - FindBinding(program.bindings, DescriptorBindingKind::StorageUint2DArray); - const auto* samplers = FindBinding(program.bindings, DescriptorBindingKind::Samplers); - const auto* shader_data = FindBinding(program.bindings, DescriptorBindingKind::UserData); - const auto shader_data_dwords = program.bindings.ShaderDataDwords(); - Check(program.bindings.descriptor_set == 3 && buffers != nullptr && sampled2d != nullptr && - sampled_array != nullptr && storage3d != nullptr && storage_uint_array != nullptr && - samplers != nullptr && buffers->binding == 0 && sampled2d->binding == 1 && - sampled_array->binding == 2 && storage3d->binding == 3 && - storage_uint_array->binding == 4 && samplers->binding == 5 && - buffers->resources == std::vector {0} && - sampled2d->resources == std::vector {0} && - sampled_array->resources == std::vector {1} && - storage3d->resources == std::vector {2} && - storage_uint_array->resources == std::vector {3} && - samplers->resources == std::vector {0} && - !program.bindings.user_data_registers.empty() && - program.bindings.buffer_offset_dword == program.bindings.user_data_registers.size() && - program.bindings.buffer_offset_count == 1 && - ((program.bindings.push_constant_size == shader_data_dwords * sizeof(uint32_t) && - shader_data == nullptr) || - (program.bindings.push_constant_size == 0 && shader_data != nullptr && - shader_data->binding == 6)) && - FindBinding(program.bindings, DescriptorBindingKind::FlattenedSrt) == nullptr && - program.binding_layout_complete, - "native binding allocator did not preserve dense typed resource groups"); -} - -void TestNativeBindingLayoutOneDimensionalImages() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - insts.push_back(ImageUse(8, Opcode::ImageSample, ResourceKind::Image, - Decoder::ImageDimension::Dim1D, 0, 2)); - insts.push_back(ImageUse(12, Opcode::ImageSample, ResourceKind::Image, - Decoder::ImageDimension::Dim1DArray, 0, 2)); - insts.push_back(ImageUse(16, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim1D, 4)); - insts.push_back(ImageUse(20, Opcode::ImageStore, ResourceKind::StorageImageUint, - Decoder::ImageDimension::Dim1DArray, 6)); - Prepare(program); - - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - std::string error; - Check(CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - const auto* sampled = FindBinding(program.bindings, DescriptorBindingKind::Sampled1D); - const auto* sampled_array = - FindBinding(program.bindings, DescriptorBindingKind::Sampled1DArray); - const auto* storage = FindBinding(program.bindings, DescriptorBindingKind::Storage1D); - const auto* storage_uint_array = - FindBinding(program.bindings, DescriptorBindingKind::StorageUint1DArray); - Check(sampled != nullptr && sampled_array != nullptr && storage != nullptr && - storage_uint_array != nullptr && sampled->resources == std::vector {0} && - sampled_array->resources == std::vector {1} && - storage->resources == std::vector {2} && - storage_uint_array->resources == std::vector {3}, - "binding allocator did not preserve first-class 1D image groups"); -} - -void TestNativeBindingLayoutSrtAndUserDataOverflow() { - Program srt; - srt.stage = ShaderType::Compute; - srt.blocks.resize(1); - for (uint32_t i = 0; i < 4; i++) { - srt.blocks[0].instructions.push_back(ScalarLoad(i * 4, i, 16, i * 4)); - } - srt.blocks[0].instructions.push_back(BufferUse(0x20, 0)); - Prepare(srt); - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - std::string error; - Check(CollectShaderInfo(srt, {.compute = &compute}, &error) && - AllocateBindings(srt, {}, &error), - error.c_str()); - const auto* flat = FindBinding(srt.bindings, DescriptorBindingKind::FlattenedSrt); - Check(flat != nullptr && flat->binding == 1 && flat->resources.empty(), - "flattened SRT did not receive one native backend binding"); - - Program overflow; - overflow.stage = ShaderType::Compute; - overflow.blocks.resize(1); - for (uint32_t i = 0; i < 33; i++) { - Instruction direct; - direct.pc = i * 4; - direct.op = Opcode::MoveU32; - direct.dst.kind = OperandKind::Register; - direct.dst.reg = {RegisterFile::Vector, i}; - direct.src[0] = Sgpr(i); - direct.src_count = 1; - overflow.blocks[0].instructions.push_back(direct); - } - Check(BuildScalarProvenance(overflow, &error) && BuildSrtPlan(overflow, &error) && - PatchSrtReads(overflow, &error) && TrackResources(overflow, &error) && - CollectShaderInfo(overflow, {.compute = &compute}, &error) && - AllocateBindings(overflow, {}, &error), - error.c_str()); - const auto* user_data = FindBinding(overflow.bindings, DescriptorBindingKind::UserData); - Check(overflow.bindings.user_data_registers.size() == 33 && - overflow.bindings.user_data_registers.front() == 0 && - overflow.bindings.user_data_registers.back() == 32 && - overflow.bindings.push_constant_size == 0 && user_data != nullptr && - user_data->binding == 0, - "oversized sparse user data was not moved to a descriptor binding"); -} - -void TestNativeBindingLayoutGds() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - Instruction append; - append.op = Opcode::DsAppend; - append.memory.kind = ResourceKind::Gds; - program.blocks[0].instructions.push_back(append); - - ShaderComputeInputInfo compute; - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error) && TrackResources(program, &error) && - CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - const auto* gds = FindBinding(program.bindings, DescriptorBindingKind::Gds); - Check(gds != nullptr && gds->binding == 0 && gds->resources.empty(), - "GDS append/consume did not allocate one native storage binding"); - - Program lds; - lds.stage = ShaderType::Compute; - lds.blocks.resize(1); - append.memory.kind = ResourceKind::Lds; - lds.blocks[0].instructions.push_back(append); - Check(BuildScalarProvenance(lds, &error) && BuildSrtPlan(lds, &error) && - PatchSrtReads(lds, &error) && TrackResources(lds, &error) && - CollectShaderInfo(lds, {.compute = &compute}, &error) && - AllocateBindings(lds, {}, &error), - error.c_str()); - Check(FindBinding(lds.bindings, DescriptorBindingKind::Gds) == nullptr, - "LDS append/consume incorrectly allocated a GDS binding"); -} - -void TestNativeBindingLayoutIsTransactional() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - std::string error; - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error) && TrackResources(program, &error) && - CollectShaderInfo(program, {.compute = &compute}, &error), - error.c_str()); - BindingLayout sentinel; - sentinel.descriptor_set = 99; - program.bindings = sentinel; - Check(!AllocateBindings(program, {.push_constant_offset = 129}, &error) && - error.find("Vulkan minimum") != std::string::npos && program.bindings == sentinel && - !program.binding_layout_complete, - "failed native binding allocation partially changed the program"); - Check(!AllocateBindings(program, {.push_constant_offset = 2}, &error) && - error.find("dword aligned") != std::string::npos && program.bindings == sentinel && - !program.binding_layout_complete, - "misaligned push-constant allocation partially changed the program"); - Check(AllocateBindings(program, {}, &error), error.c_str()); - const auto layout = program.bindings; - Check(!AllocateBindings(program, {}, &error) && - error.find("already allocated") != std::string::npos && program.bindings == layout, - "repeated native binding allocation was accepted or mutated layout"); - - Program tail; - tail.stage = ShaderType::Compute; - tail.blocks.resize(1); - Instruction direct; - direct.op = Opcode::MoveU32; - direct.dst.kind = OperandKind::Register; - direct.dst.reg = {RegisterFile::Vector, 0}; - direct.src[0] = Sgpr(0); - direct.src_count = 1; - tail.blocks[0].instructions = {direct}; - Check(BuildScalarProvenance(tail, &error) && BuildSrtPlan(tail, &error) && - PatchSrtReads(tail, &error) && TrackResources(tail, &error) && - CollectShaderInfo(tail, {.compute = &compute}, &error) && - AllocateBindings(tail, {.push_constant_offset = 124}, &error) && - tail.bindings.push_constant_size == 4 && - FindBinding(tail.bindings, DescriptorBindingKind::UserData) == nullptr, - "valid final dword of the guaranteed push-constant range was rejected"); -} - -void TestNativeBindingLayoutTracksReachingUserData() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - auto& insts = program.blocks[0].instructions; - insts.push_back(Move(0, 100, 4)); - Instruction forwarded; - forwarded.pc = 4; - forwarded.op = Opcode::MoveU32; - forwarded.dst.kind = OperandKind::Register; - forwarded.dst.reg = {RegisterFile::Vector, 0}; - forwarded.src[0] = Sgpr(100); - forwarded.src_count = 1; - insts.push_back(forwarded); - auto sparse = forwarded; - sparse.pc = 8; - sparse.dst.reg.index = 1; - sparse.src[0] = Sgpr(20); - insts.push_back(sparse); - insts.push_back(MoveImmediate(12, 101, 7)); - auto constant = forwarded; - constant.pc = 16; - constant.dst.reg.index = 2; - constant.src[0] = Sgpr(101); - insts.push_back(constant); - - std::string error; - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error) && TrackResources(program, &error) && - CollectShaderInfo(program, {.compute = &compute}, &error), - error.c_str()); - auto fallback = program; - Check(AllocateBindings(program, {}, &error) && - program.bindings.user_data_registers == std::vector({4, 20}) && - program.bindings.push_constant_size == 8 && - FindBinding(program.bindings, DescriptorBindingKind::UserData) == nullptr, - "native user-data map included a temporary or missed sparse roots"); - Check(AllocateBindings(fallback, {.max_push_dwords = 1}, &error) && - fallback.bindings.user_data_registers == std::vector({4, 20}) && - fallback.bindings.push_constant_size == 0 && - FindBinding(fallback.bindings, DescriptorBindingKind::UserData) != nullptr, - "max push-dword limit did not move exact sparse user data to storage"); -} - -void TestNativeBindingLayoutRejectsUnknownShapeAndBadProvenance() { - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - std::string error; - - Program image; - image.stage = ShaderType::Compute; - image.blocks.resize(1); - image.blocks[0].instructions = {ImageUse(4, Opcode::ImageStore, ResourceKind::StorageImage, - Decoder::ImageDimension::Dim2D)}; - Prepare(image); - Check(CollectShaderInfo(image, {.compute = &compute}, &error), error.c_str()); - image.info.images[0].dimension = Decoder::ImageDimension::Unknown; - image.bindings.descriptor_set = 77; - const auto image_layout = image.bindings; - Check(!AllocateBindings(image, {}, &error) && - error.find("invalid image binding class") != std::string::npos && - image.bindings == image_layout && !image.binding_layout_complete, - "unknown image shape was defaulted or partially allocated"); - - Program provenance; - provenance.stage = ShaderType::Compute; - provenance.blocks.resize(1); - Instruction direct; - direct.op = Opcode::MoveU32; - direct.dst.kind = OperandKind::Register; - direct.dst.reg = {RegisterFile::Vector, 0}; - direct.src[0] = Sgpr(4); - direct.src_count = 1; - provenance.blocks[0].instructions = {direct}; - Check(BuildScalarProvenance(provenance, &error) && BuildSrtPlan(provenance, &error) && - PatchSrtReads(provenance, &error) && TrackResources(provenance, &error) && - CollectShaderInfo(provenance, {.compute = &compute}, &error), - error.c_str()); - provenance.blocks[0].instructions[0].scalar_sources[0] = - static_cast(provenance.provenance.values.size() + 1); - provenance.bindings.descriptor_set = 88; - const auto provenance_layout = provenance.bindings; - Check(!AllocateBindings(provenance, {}, &error) && - error.find("invalid scalar provenance reference") != std::string::npos && - provenance.bindings == provenance_layout && !provenance.binding_layout_complete, - "malformed per-use provenance was ignored or partially allocated"); -} - -void TestNativeBindingLayoutDynamicSrtDoesNotUseFlatBinding() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - for (uint32_t i = 0; i < 4; i++) { - auto load = ScalarLoad(i * 4, i, 16, i * 4); - load.src[0] = Sgpr(20); - load.src_count = 1; - program.blocks[0].instructions.push_back(load); - } - program.blocks[0].instructions.push_back(BufferUse(0x20, 0)); - Prepare(program); - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - std::string error; - Check(program.srt.reads.empty() && program.srt.dynamic_reads.size() == 4 && - CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - Check(program.bindings.user_data_registers == std::vector({16, 17, 20}) && - FindBinding(program.bindings, DescriptorBindingKind::FlattenedSrt) == nullptr, - "dynamic-only SRT reads were flattened or lost reaching user data"); -} - -void TestNativeBindingLayoutTracksRawScalarMemoryBase() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - auto load = ScalarLoad(0, 0, 16, 0); - load.src[0] = Sgpr(20); - load.src_count = 1; - Instruction consume; - consume.pc = 4; - consume.op = Opcode::MoveU32; - consume.dst.kind = OperandKind::Register; - consume.dst.reg = {RegisterFile::Vector, 0}; - consume.src[0] = Sgpr(0); - consume.src_count = 1; - program.blocks[0].instructions = {load, consume}; - - std::string error; - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - program.srt.reads.empty() && program.srt.dynamic_reads.empty() && - PatchSrtReads(program, &error) && TrackResources(program, &error) && - CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - const auto* address_memory = - FindBinding(program.bindings, DescriptorBindingKind::AddressMemory); - Check(program.blocks[0].instructions[0].op == Opcode::SLoadDword && - program.bindings.user_data_registers == std::vector({16, 17, 20}) && - FindBinding(program.bindings, DescriptorBindingKind::FlattenedSrt) == nullptr && - address_memory != nullptr && - address_memory->resources == std::vector({0}) && - program.info.addresses.size() == 1 && - program.info.addresses[0].kind == ResourceKind::ScalarBuffer, - "ordinary raw scalar memory omitted its implicit base or offset roots"); -} - -void TestRawScalarMemoryTracksReachingBaseIdentity() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - auto first = ScalarLoad(8, 0, 16, 0); - first.memory.offset = static_cast(-4); - first.src[0] = Sgpr(20); - first.src_count = 1; - auto second = ScalarLoad(20, 1, 16, 0); - second.src[0] = Sgpr(20); - second.src_count = 1; - program.blocks[0].instructions = { - MoveImmediate(0, 16, 0x1000), MoveImmediate(4, 17, 0), first, - MoveImmediate(12, 16, 0x2000), MoveImmediate(16, 17, 0), second}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error) && - PatchSrtReads(program, &error) && TrackResources(program, &error), - error.c_str()); - Check(program.info.addresses.size() == 2 && - program.info.addresses[0].source != program.info.addresses[1].source && - program.blocks[0].instructions[2].memory.resource == 0 && - program.blocks[0].instructions[5].memory.resource == 1, - "raw scalar loads with redefined SBASE collapsed to one address " - "resource"); - - std::array user_data {}; - user_data[20] = 4; - ResourceSnapshot snapshot; - Check(MaterializeResources(program, {user_data}, snapshot, &error), error.c_str()); - Check(snapshot.addresses.size() == 2 && snapshot.addresses[0].guest_base == 0x1000 && - snapshot.addresses[0].binding_base == 0x0ffc && - snapshot.addresses[1].guest_base == 0x2000 && - snapshot.addresses[1].binding_base == 0x2000, - "runtime snapshot lost per-use raw scalar base values"); -} - -void TestNativeBindingLayoutUsesExplicitFlatMemory() { - Program program; - program.stage = ShaderType::Compute; - program.blocks.resize(1); - Instruction load; - load.op = Opcode::FlatLoadDword; - load.memory.kind = ResourceKind::Flat; - program.blocks[0].instructions = {load}; - Prepare(program); - - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - std::string error; - Check(CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - const auto* flat = FindBinding(program.bindings, DescriptorBindingKind::AddressMemory); - Check(flat != nullptr && flat->resources == std::vector({0}) && - program.info.addresses.size() == 1 && - program.info.addresses[0].kind == ResourceKind::Flat && - FindBinding(program.bindings, DescriptorBindingKind::Buffers) == nullptr, - "flat address space was aliased to an ordinary buffer descriptor group"); - - ResourceSnapshot snapshot; - SrtRuntime runtime; - runtime.flat_memory_base = 0x1234567887654000ull; - Check(MaterializeResources(program, runtime, snapshot, &error) && - snapshot.addresses.size() == 1 && - snapshot.addresses[0].guest_base == 0x1234567887654000ull && - snapshot.addresses[0].binding_base == 0x1234567887654000ull, - "unbased flat virtual address space lost its runtime binding base"); -} - -void TestNativeBindingLayoutHonorsUserDataCount() { - Program program; - program.stage = ShaderType::Compute; - program.user_data_count = 8; - program.blocks.resize(1); - for (uint32_t i = 0; i < 2; i++) { - Instruction direct; - direct.pc = i * 4; - direct.op = Opcode::MoveU32; - direct.dst.kind = OperandKind::Register; - direct.dst.reg = {RegisterFile::Vector, i}; - direct.src[0] = Sgpr(7 + i); - direct.src_count = 1; - program.blocks[0].instructions.push_back(direct); - } - std::string error; - ShaderComputeInputInfo compute; - compute.thread_ids_num = 1; - Check(BuildScalarProvenance(program, &error) && - program.provenance.values[program.blocks[0].instructions[0].scalar_sources[0]].op == - ScalarValueOp::UserData && - program.blocks[0].instructions[1].scalar_sources[0] == ScalarProvenance::Unknown && - BuildSrtPlan(program, &error) && PatchSrtReads(program, &error) && - TrackResources(program, &error) && - CollectShaderInfo(program, {.compute = &compute}, &error) && - AllocateBindings(program, {}, &error) && - program.bindings.user_data_registers == std::vector({7}), - "stage user-data count did not bound provenance roots"); - - Program invalid; - invalid.user_data_count = 65; - invalid.blocks.resize(1); - Check(!BuildScalarProvenance(invalid, &error) && error.find("exceeds 64") != std::string::npos, - "out-of-range stage user-data count was accepted"); -} - -void TestNativeBindingLayoutHonorsUserDataBase() { - Program program; - program.stage = ShaderType::Vertex; - program.user_data_base = 8; - program.user_data_count = 8; - program.blocks.resize(1); - program.blocks[0].instructions = {Move(0, 20, 7), Move(4, 21, 8), Move(8, 22, 15), - Move(12, 23, 16), BufferUse(16, 8)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && - program.blocks[0].instructions[0].scalar_sources[0] == ScalarProvenance::Unknown && - program.provenance.values[program.blocks[0].instructions[1].scalar_sources[0]].op == - ScalarValueOp::UserData && - program.blocks[0].instructions[3].scalar_sources[0] == ScalarProvenance::Unknown && - BuildSrtPlan(program, &error), - error.c_str()); - - const std::array user_data = {0x11111111, 0x22222222, 0x33333333, 0x44444444, - 0, 0, 0, 0xaaaaaaaa}; - DescriptorValue descriptor; - const auto source = program.blocks[0].instructions.back().memory.resource_source; - Check(EvaluateDescriptorSource(program, source, 16, {user_data}, descriptor, &error) && - descriptor.dwords[0] == user_data[0] && descriptor.dwords[3] == user_data[3], - "vertex user-data base was not translated to runtime-local indices"); - - ShaderVertexInputInfo vertex; - Check(PatchSrtReads(program, &error) && TrackResources(program, &error) && - CollectShaderInfo(program, {.vertex = &vertex}, &error) && - AllocateBindings(program, {}, &error), - error.c_str()); - Check(program.bindings.user_data_registers == std::vector({8, 9, 10, 11, 15}), - "native binding plan did not retain physical shifted user-SGPR indices"); -} - -void TestTextureNullDescriptorUsesAddressBits() { - ShaderTextureResource texture; - const uint32_t captured[8] = {0x00000000, 0xc3800000, 0x0059c09f, 0x91b00fac, - 0x00000000, 0x00700000, 0x00000000, 0x00000000}; - std::copy(std::begin(captured), std::end(captured), std::begin(texture.fields)); - Check(texture.IsNull(), - "zero-address image with populated metadata was not classified as null"); - texture.fields[0] = 1; - Check(!texture.IsNull(), "nonzero image base address was classified as null"); +void TestResourceLimitIsTransactional() { + Fixture fixture; + MemoryInfo memory; + memory.kind = ResourceKind::Buffer; + for (uint32_t index = 0; index <= ShaderInfo::MaxBuffers; index++) { + const auto handle = fixture.Buffer( + {Value(index), Value(index + 1u), Value(index + 2u), Value(index + 3u)}, + index * 4u); + fixture.Emit(ValueOpcode::LoadBufferU32, + {handle, Value(0u), Value(0u), Value(0u), Value(true)}, + fixture.AddMemory(memory, index * 4u)); + } + std::string error; + Check(BuildSrtPlan(fixture.program, &error), + "SRT plan failed before resource-limit test"); + Check(!TrackResources(fixture.program, &error) && + error.find("buffer resource limit exceeded") != std::string::npos && + !fixture.program.resource_tracking_complete && + fixture.program.info.buffers.empty() && + fixture.program.values->descriptor_sources.empty(), + "resource-limit failure partially mutated typed resource state"); } } // namespace int main() { - const char* current = "startup"; -#define RUN(test) \ - current = #test; \ - test() - try { - RUN(TestDenseBufferPatching); - RUN(TestScalarAndVectorBufferAlias); - RUN(TestBufferImageAliasIsLinkedDuringTracking); - RUN(TestImagesAndSamplers); - RUN(TestDynamicPhiResource); - RUN(TestTrackingRequiresCompletedSrtPlan); - RUN(TestCyclicResourceIsRejected); - RUN(TestUnknownSourceFailsWithoutPatching); - RUN(TestResourceLimitFailsTransactionally); - RUN(TestComputeShaderInfoCollection); - RUN(TestVertexShaderInfoCollection); - RUN(TestPixelShaderInfoCollection); - RUN(TestShaderInfoCollectionIsTransactional); - RUN(TestShaderInfoMetadataValidation); - RUN(TestTrackingRequiresSrtPatching); - RUN(TestDynamicSrtReadRemainsExplicit); - RUN(TestSrtPatchingFailureIsTransactional); - RUN(TestScalarMemoryGroupsSnapshotOperands); - RUN(TestSrtPatchingHandlesGvnAndMoveForwarding); - RUN(TestSrtPatchingHandlesCfgProducers); - RUN(TestSrtPatchPlanValidation); - RUN(TestMaterializationSharesReadConstEvaluation); - RUN(TestInvalidImagesMaterializeAsNull); - RUN(TestInvalidBuffersMaterializeAsNull); - RUN(TestMaterializationFailureIsTransactional); - RUN(TestResourceSpecializationIsTypedAndTransactional); - RUN(TestRuntimeSpecializationCoversBakedBufferAndAddressFields); - RUN(TestTrackedProgramIsImmutable); - RUN(TestNativeBindingLayout); - RUN(TestNativeBindingLayoutOneDimensionalImages); - RUN(TestNativeBindingLayoutSrtAndUserDataOverflow); - RUN(TestNativeBindingLayoutGds); - RUN(TestNativeBindingLayoutIsTransactional); - RUN(TestNativeBindingLayoutTracksReachingUserData); - RUN(TestNativeBindingLayoutRejectsUnknownShapeAndBadProvenance); - RUN(TestNativeBindingLayoutDynamicSrtDoesNotUseFlatBinding); - RUN(TestNativeBindingLayoutTracksRawScalarMemoryBase); - RUN(TestRawScalarMemoryTracksReachingBaseIdentity); - RUN(TestNativeBindingLayoutUsesExplicitFlatMemory); - RUN(TestNativeBindingLayoutHonorsUserDataCount); - RUN(TestNativeBindingLayoutHonorsUserDataBase); - RUN(TestTextureNullDescriptorUsesAddressBits); - std::cout << "ResourceTrackingTests: all cases passed\n"; - return 0; - } catch (const std::exception& e) { - std::cerr << "ResourceTrackingTests: " << current << " failed: " << e.what() << '\n'; - return 1; - } -#undef RUN + try { + const auto Run = [](const char *name, auto test) { + try { + test(); + } catch (const std::exception &exception) { + throw std::runtime_error(std::string(name) + ": " + exception.what()); + } + }; + Run("dense buffers", TestDenseBufferTracking); + Run("scalar/vector alias", TestScalarAndVectorBufferAlias); + Run("images and samplers", TestImagesSamplersAndAliases); + Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization); + Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); + Run("phi validation", TestPhiValidation); + Run("invariant loop phi", TestInvariantLoopPhi); + Run("address materialization", TestAddressMaterializationAndSpecialization); + Run("shader info and bindings", TestShaderInfoAndBindingLayout); + Run("resource limit", TestResourceLimitIsTransactional); + } catch (const std::exception &exception) { + std::cerr << "resource tracking test failed: " << exception.what() << '\n'; + return 1; + } + std::cout << "resource tracking tests passed\n"; + return 0; } + +// The full emulator supplies these assertion hooks through common. This focused +// target links only fmt; keep assertion failures observable without widening +// its legacy build manifest. +namespace Common { +int DbgExitIfHandler(const char *expression, const char *file, int line) { + throw std::runtime_error(std::string("typed IR assertion: ") + expression + + " at " + file + ':' + std::to_string(line)); +} + +void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); } +} // namespace Common + +// Keep this focused standalone target self-contained by amalgamating its small +// typed-IR implementation set. +#include "graphics/shader/recompiler/ir/Block.cpp" +#include "graphics/shader/recompiler/ir/Type.cpp" +#include "graphics/shader/recompiler/ir/Value.cpp" +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" +#include "graphics/shader/recompiler/ir/ValueProgram.cpp" diff --git a/tests/ScalarProvenanceTests.cpp b/tests/ScalarProvenanceTests.cpp index 4cf19330c..ee8bca54d 100644 --- a/tests/ScalarProvenanceTests.cpp +++ b/tests/ScalarProvenanceTests.cpp @@ -1,1231 +1,420 @@ -#include "graphics/shader/recompiler/ir/ReadLaneElimination.h" -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" -#include "graphics/shader/recompiler/ir/SrtWalker.h" +#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" +#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" -#include +#include #include #include +#include namespace { using namespace Libs::Graphics::ShaderRecompiler::IR; using Libs::Graphics::ShaderType; -void Check(bool condition, const char* message) { - if (!condition) { - throw std::runtime_error(message); - } +void Check(bool condition, const char *message) { + if (!condition) { + throw std::runtime_error(message); + } } -Operand Sgpr(uint32_t reg) { - Operand operand; - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Scalar; - operand.reg.index = reg; - return operand; -} +struct Fixture { + Program program; -Operand Vgpr(uint32_t reg) { - Operand operand; - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::Vector; - operand.reg.index = reg; - return operand; -} + explicit Fixture(uint32_t block_count = 1) { + program.stage = ShaderType::Compute; + program.shader_hash = 0x12345678u; + program.user_data_base = 2; + program.values = std::make_shared(); + for (uint32_t index = 0; index < block_count; index++) { + program.values->block_storage.push_back(std::make_unique()); + auto *block = program.values->block_storage.back().get(); + program.values->blocks.push_back(block); + program.values->block_info.push_back({.id = index}); + } + } -Operand Imm(uint32_t value) { - Operand operand; - operand.kind = OperandKind::ImmediateU32; - operand.imm = value; - return operand; -} + Block &BlockAt(uint32_t index = 0) { return *program.values->blocks[index]; } -Operand M0() { - Operand operand; - operand.kind = OperandKind::Register; - operand.reg.file = RegisterFile::M0; - return operand; -} + Value Emit(ValueOpcode opcode, std::initializer_list args = {}, + uint64_t flags = 0, uint32_t block = 0) { + return Value(&BlockAt(block).AppendNewInst(opcode, args, flags)); + } -Instruction Move(uint32_t pc, uint32_t dst, uint32_t src) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::MoveU32; - inst.dst = Sgpr(dst); - inst.src[0] = Sgpr(src); - inst.src_count = 1; - return inst; -} + Value EmitMemory(ValueOpcode opcode, std::initializer_list args, + uint32_t memory, uint32_t pc = 0x40, uint32_t block = 0) { + MemoryFlags flags{.index = memory, .pc = pc}; + uint64_t bits = 0; + std::memcpy(&bits, &flags, sizeof(flags)); + return Emit(opcode, args, bits, block); + } -Instruction MoveImmediate(uint32_t pc, uint32_t dst, uint32_t value) { - auto inst = Move(pc, dst, 0); - inst.src[0] = Imm(value); - return inst; -} + uint32_t AddMemory(ResourceKind kind, int32_t offset = 0) { + MemoryInfo info; + info.kind = kind; + info.offset = static_cast(offset); + program.values->memory_info.push_back(info); + return static_cast(program.values->memory_info.size() - 1u); + } -Instruction Move64(uint32_t pc, uint32_t dst, uint32_t src) { - auto inst = Move(pc, dst, src); - inst.op = Opcode::MoveU64; - return inst; -} - -Instruction WriteLane(uint32_t pc, uint32_t dst, uint32_t src, uint32_t lane) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::WriteLaneU32; - inst.dst = Vgpr(dst); - inst.src[0] = Sgpr(src); - inst.src[1] = Imm(lane); - inst.src_count = 2; - return inst; -} - -Instruction ReadLane(uint32_t pc, uint32_t dst, uint32_t src, uint32_t lane) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::ReadLaneU32; - inst.dst = Sgpr(dst); - inst.src[0] = Vgpr(src); - inst.src[1] = Imm(lane); - inst.src_count = 2; - return inst; -} - -Instruction BufferUse(uint32_t pc, uint32_t base_sgpr) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::BufferLoadDword; - inst.memory.kind = ResourceKind::Buffer; - inst.memory.resource = base_sgpr / 4; - return inst; -} - -const ScalarValue& Value(const Program& program, uint32_t id) { - Check(id < program.provenance.values.size(), "invalid scalar value ID"); - return program.provenance.values[id]; -} - -bool ReadHostMemory(void*, uint64_t address, uint32_t* value) { - if (address == 0 || value == nullptr) { - return false; - } - std::memcpy(value, reinterpret_cast(address), sizeof(*value)); - return true; -} - -struct ReadCounter { - uint32_t reads = 0; + void Plan() { + std::string error; + Check(BuildSrtPlan(program, &error), error.c_str()); + } }; -bool CountedHostMemory(void* userdata, uint64_t address, uint32_t* value) { - auto* counter = static_cast(userdata); - if (counter != nullptr) { - counter->reads++; - } - return ReadHostMemory(nullptr, address, value); +struct TestMemory { + std::unordered_map words; + uint32_t reads = 0; +}; + +bool ReadMemory(void *userdata, uint64_t address, uint32_t *value) { + auto &memory = *static_cast(userdata); + const auto it = memory.words.find(address); + if (it == memory.words.end()) { + return false; + } + *value = it->second; + memory.reads++; + return true; } -void SetPointer(std::vector* user_data, uint32_t reg, const void* pointer) { - const auto address = reinterpret_cast(pointer); - (*user_data)[reg] = static_cast(address); - (*user_data)[reg + 1u] = static_cast(address >> 32u); +Value Address(Fixture &fixture, Value low, Value high, uint32_t block = 0) { + return fixture.Emit(ValueOpcode::GetAddressResource, {low, high}, 0, block); } -void TestPerUseDescriptorDefinitions() { - Program program; - program.blocks.resize(1); - auto& instructions = program.blocks[0].instructions; - instructions.push_back(BufferUse(0, 0)); - for (uint32_t i = 0; i < 4; i++) { - instructions.push_back(Move(4 + i * 4, i, 20 + i)); - } - instructions.push_back(BufferUse(20, 0)); - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* before = GetDescriptorSource(program, instructions[0].memory.resource_source); - const auto* after = GetDescriptorSource(program, instructions[5].memory.resource_source); - Check(before != nullptr && after != nullptr, "descriptor sources were not attached"); - Check(*before != *after, "descriptor source was incorrectly shared across SGPR redefinition"); - for (uint32_t i = 0; i < 4; i++) { - Check(Value(program, before->dwords[i]).op == ScalarValueOp::UserData && - Value(program, before->dwords[i]).imm == i, - "pre-copy descriptor does not use the reaching user-data definition"); - Check(Value(program, after->dwords[i]).op == ScalarValueOp::UserData && - Value(program, after->dwords[i]).imm == 20 + i, - "post-copy descriptor does not use the copied user-data definition"); - } +Value RawRead(Fixture &fixture, Value address, Value offset, uint32_t memory, + uint32_t block = 0) { + return fixture.EmitMemory(ValueOpcode::LoadAddressU32, + {address, offset, Value(0u), Value(true)}, memory, + 0x80, block); } -void TestCfgPhi() { - Program program; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - program.blocks[1].instructions.push_back(MoveImmediate(4, 0, 0x11111111)); - program.blocks[2].instructions.push_back(MoveImmediate(8, 0, 0x22222222)); - program.blocks[3].instructions.push_back(BufferUse(12, 0)); +void TestImmediateFlatteningAndGvn() { + Fixture fixture; + const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer, 0x20); + const auto first = RawRead( + fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory); + const auto second = RawRead( + fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory); + fixture.Emit(ValueOpcode::GetBufferResource, + {first, second, Value(16u), Value(0u)}); - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[3].instructions[0].memory.resource_source); - Check(source != nullptr, "merged descriptor source was not attached"); - const auto& phi = Value(program, source->dwords[0]); - Check(phi.op == ScalarValueOp::Phi && phi.phi_args.size() == 2, - "differing CFG definitions did not create a provenance phi"); - Check(Value(program, phi.phi_args[0]).op == ScalarValueOp::Constant && - Value(program, phi.phi_args[1]).op == ScalarValueOp::Constant, - "provenance phi does not reference both constant definitions"); - Check(BuildSrtPlan(program, &error) && program.srt.dynamic_sources.size() == 1 && - program.srt.dynamic_sources[0] == - program.blocks[3].instructions[0].memory.resource_source, - "acyclic control-flow descriptor phi was not classified dynamic"); + fixture.Plan(); + Check(fixture.program.values->srt_reads.size() == 1, + "equivalent typed scalar reads were not coalesced"); + Check(fixture.program.values->dynamic_reads.empty(), + "immediate scalar read was classified as dynamic"); + Check(fixture.program.values->memory_info[memory].planning_only, + "flattened raw read was not kept as a planning-only root"); + + TestMemory memory_image{{{0x1020u, 0xfeedbeefu}}}; + SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + std::vector flat; + std::string error; + Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str()); + Check(flat == std::vector{0xfeedbeefu} && memory_image.reads == 1, + "flattened SRT did not evaluate its canonical read once"); } -void TestNestedLoopPhiConvergence() { - Program program; - program.blocks.resize(4); - program.blocks[0].predecessors = {1}; - program.blocks[0].successors = {1}; - program.blocks[1].predecessors = {0, 3}; - program.blocks[1].successors = {0, 2}; - program.blocks[2].predecessors = {1}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {2}; - program.blocks[3].successors = {1}; - Instruction increment; - increment.op = Opcode::IAddU32; - increment.dst = Sgpr(0); - increment.src[0] = Sgpr(0); - increment.src[1] = Imm(1); - increment.src_count = 2; - program.blocks[0].instructions = {increment}; - program.blocks[2].instructions = {BufferUse(4, 0)}; +void TestDynamicReadRemainsTyped() { + Fixture fixture; + const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer); + const auto offset = fixture.Emit(ValueOpcode::GetUserData, + {Value(static_cast(2))}); + const auto read = RawRead( + fixture, Address(fixture, Value(0x1000u), Value(0u)), offset, memory); + fixture.Emit(ValueOpcode::GetBufferResource, + {read, Value(0u), Value(16u), Value(0u)}); - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[2].instructions[0].memory.resource_source); - Check(source != nullptr, "nested-loop descriptor source was not attached"); - const auto value_id = source->dwords[0]; - const auto& phi = Value(program, value_id); - Check(phi.op == ScalarValueOp::Phi && phi.phi_args.size() == 2 && - ((phi.phi_args[0] == value_id && phi.phi_args[1] != value_id) || - (phi.phi_args[1] == value_id && phi.phi_args[0] != value_id)), - "nested loop did not retain its recursive scalar provenance phi"); -} - -void TestDiamondReadPathsAreDynamic() { - std::array left = {0x11111111u}; - std::array right = {0x22222222u}; - Program program; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - const auto load = [](uint32_t pc, uint32_t base) { - Instruction inst; - inst.pc = pc; - inst.op = Opcode::SLoadDword; - inst.dst = Sgpr(0); - inst.src[0] = Imm(0); - inst.src_count = 1; - inst.memory.kind = ResourceKind::ScalarBuffer; - inst.memory.resource = base; - return inst; - }; - program.blocks[1].instructions = {load(4, 4)}; - program.blocks[2].instructions = {load(8, 6)}; - program.blocks[3].instructions = {BufferUse(12, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const auto source = program.blocks[3].instructions[0].memory.resource_source; - Check(program.srt.reads.empty() && program.srt.dynamic_reads.empty() && - program.srt.dynamic_sources.size() == 1 && program.srt.dynamic_sources[0] == source, - "inactive diamond read paths were incorrectly flattened"); - std::array user_data = {}; - const auto left_address = reinterpret_cast(left.data()); - const auto right_address = reinterpret_cast(right.data()); - user_data[4] = static_cast(left_address); - user_data[5] = static_cast(left_address >> 32u); - user_data[6] = static_cast(right_address); - user_data[7] = static_cast(right_address >> 32u); - ReadCounter counter; - DescriptorValue descriptor; - const SrtRuntime runtime {user_data, 0, CountedHostMemory, &counter}; - Check(!EvaluateDescriptorSource(program, source, 12, runtime, descriptor, &error) && - counter.reads == 0 && error.find("GPU-dynamic") != std::string::npos, - "dynamic descriptor evaluation touched an inactive read path"); -} - -void TestEquivalentConstantPhiIsStatic() { - Program program; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - Instruction add; - add.op = Opcode::IAddU32; - add.dst = Sgpr(0); - add.src[0] = Imm(1); - add.src[1] = Imm(1); - add.src_count = 2; - program.blocks[1].instructions = {add}; - program.blocks[2].instructions = {MoveImmediate(8, 0, 2)}; - program.blocks[3].instructions = {BufferUse(12, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const auto source = program.blocks[3].instructions[0].memory.resource_source; - Check(program.srt.dynamic_sources.empty(), - "equivalent constant phi was unnecessarily classified dynamic"); - const std::array user_data = {}; - DescriptorValue descriptor; - const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; - Check(EvaluateDescriptorSource(program, source, 12, runtime, descriptor, &error) && - descriptor.dwords[0] == 2, - "equivalent constant phi did not evaluate as one static value"); -} - -void TestWideMoveInvalidatesAndCopiesBothDwords() { - Program program; - program.blocks.resize(1); - auto& instructions = program.blocks[0].instructions; - instructions.push_back(Move64(0, 0, 20)); - instructions.push_back(Move64(4, 2, 22)); - instructions.push_back(BufferUse(8, 0)); - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = GetDescriptorSource(program, instructions.back().memory.resource_source); - Check(source != nullptr && - DescriptorSourceResolved(program, instructions.back().memory.resource_source), - "wide descriptor copy did not produce a resolved source"); - for (uint32_t i = 0; i < 4; i++) { - Check(Value(program, source->dwords[i]).op == ScalarValueOp::UserData && - Value(program, source->dwords[i]).imm == 20 + i, - "wide move did not copy both scalar dwords"); - } -} - -void TestScalarCarryChain() { - Program program; - program.blocks.resize(1); - Instruction low; - low.pc = 0; - low.op = Opcode::ScalarAddCarryU32; - low.dst = Sgpr(4); - low.src[0] = Sgpr(0); - low.src[1] = Imm(16); - low.src[2] = Imm(0); - low.src_count = 3; - Instruction high = low; - high.pc = 4; - high.dst = Sgpr(5); - high.src[0] = Sgpr(1); - high.src[1] = Imm(0); - high.src[2].kind = OperandKind::Register; - high.src[2].reg.file = RegisterFile::Scc; - high.src[2].reg.index = 0; - program.blocks[0].instructions = {low, high, BufferUse(8, 4)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[0].instructions.back().memory.resource_source); - Check(source != nullptr, "carry-chain descriptor source was not attached"); - const auto& low_value = Value(program, source->dwords[0]); - const auto& high_value = Value(program, source->dwords[1]); - Check(low_value.op == ScalarValueOp::AddCarry && high_value.op == ScalarValueOp::AddCarry, - "scalar add/addc pair was not retained"); - Check(Value(program, high_value.args[2]).op == ScalarValueOp::Carry, - "high pointer dword does not consume the low add carry"); -} - -void TestReadConstDefinition() { - Program program; - program.blocks.resize(1); - Instruction load; - load.pc = 4; - load.op = Opcode::SLoadDword; - load.dst = Sgpr(16); - load.src[0] = Imm(8); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 4; - load.memory.offset = 16; - program.blocks[0].instructions.push_back(load); - program.blocks[0].instructions.push_back(BufferUse(8, 16)); - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[0].instructions[1].memory.resource_source); - Check(source != nullptr, "read-constant descriptor source was not attached"); - const auto& read = Value(program, source->dwords[0]); - Check(read.op == ScalarValueOp::ReadConst && read.imm == 16, - "raw scalar load was not represented as ReadConst"); - Check(Value(program, read.args[0]).op == ScalarValueOp::UserData && - Value(program, read.args[0]).imm == 4 && Value(program, read.args[1]).imm == 5, - "ReadConst pointer does not retain its user-data roots"); - Check(Value(program, read.args[2]).op == ScalarValueOp::Constant && - Value(program, read.args[2]).imm == 8, - "ReadConst dynamic/immediate offset operand was lost"); -} - -void TestReadConstBufferAndValueNumbering() { - Program program; - program.blocks.resize(1); - Instruction load; - load.op = Opcode::SBufferLoadDword; - load.dst = Sgpr(16); - load.src[0] = Imm(3); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 1; - load.memory.offset = 12; - program.blocks[0].instructions.push_back(load); - load.pc = 4; - load.dst = Sgpr(17); - program.blocks[0].instructions.push_back(load); - program.blocks[0].instructions.push_back(BufferUse(8, 16)); - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[0].instructions.back().memory.resource_source); - Check(source != nullptr, "ReadConstBuffer source was not attached"); - Check(source->dwords[0] == source->dwords[1], - "identical read-constant values at different PCs were not " - "value-numbered"); - const auto& read = Value(program, source->dwords[0]); - Check(read.op == ScalarValueOp::ReadConstBuffer && read.args[3] != 0 && read.args[4] != 0, - "ReadConstBuffer lost descriptor dword 3 or its offset"); -} - -void TestReadLaneDescriptorSpill() { - std::array table {}; - table[72] = 0x89abcdefu; - - Program program; - program.wave_size = 64; - program.user_data_count = 28; - program.blocks.resize(1); - for (uint32_t component = 0; component < 4; component++) { - Instruction load; - load.pc = 0x14; - load.op = Opcode::SBufferLoadDword; - load.dst = Sgpr(84 + component); - load.src[0] = Imm(0); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 6; - load.memory.offset = 288 + component * 4; - load.memory.component_index = component; - load.memory.component_count = 4; - program.blocks[0].instructions.push_back(load); - } - program.blocks[0].instructions.push_back(WriteLane(0x20, 11, 86, 2)); - program.blocks[0].instructions.push_back(WriteLane(0x44, 11, 85, 1)); - program.blocks[0].instructions.push_back(WriteLane(0x60, 11, 84, 0)); - program.blocks[0].instructions.push_back(MoveImmediate(0xb0, 84, 0)); - program.blocks[0].instructions.push_back(ReadLane(0x510, 0, 11, 0)); - Instruction sample; - sample.pc = 0x868; - sample.op = Opcode::ImageSample; - sample.memory.kind = ResourceKind::Image; - sample.memory.resource = 0; - sample.memory.sampler = 4; - sample.memory.image_dimension = - Libs::Graphics::ShaderRecompiler::Decoder::ImageDimension::Dim2D; - program.blocks[0].instructions.push_back(sample); - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const auto source_id = program.blocks[0].instructions.back().memory.resource_source; - const auto* source = GetDescriptorSource(program, source_id); - Check(source != nullptr && DescriptorSourceResolved(program, source_id), - "readlane descriptor spill was not resolved"); - Check(Value(program, source->dwords[0]).op == ScalarValueOp::ReadConstBuffer, - "readlane did not recover the pre-overwrite SRT value"); - std::vector user_data(28); - SetPointer(&user_data, 24, table.data()); - user_data[26] = sizeof(table); - DescriptorValue descriptor; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(EvaluateDescriptorSource(program, source_id, sample.pc, runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[0] == table[72], - "readlane descriptor spill evaluated the overwritten scalar value"); -} - -void TestReadLaneVectorOverwriteInvalidatesSpill() { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(1); - Instruction overwrite; - overwrite.pc = 4; - overwrite.op = Opcode::MoveU32; - overwrite.dst = Vgpr(11); - overwrite.src[0] = Imm(0); - overwrite.src_count = 1; - program.blocks[0].instructions = {WriteLane(0, 11, 4, 0), overwrite, ReadLane(8, 0, 11, 0), - BufferUse(12, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto source = program.blocks[0].instructions.back().memory.resource_source; - Check(GetDescriptorSource(program, source) != nullptr && - !DescriptorSourceResolved(program, source), - "vector overwrite left stale writelane provenance"); -} - -void TestReadLanePhi() { - Program program; - program.wave_size = 32; - program.user_data_count = 8; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - program.blocks[1].instructions = {WriteLane(0, 11, 4, 0)}; - program.blocks[2].instructions = {WriteLane(4, 11, 5, 0)}; - program.blocks[3].instructions = {ReadLane(8, 0, 11, 0), BufferUse(12, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[3].instructions.back().memory.resource_source); - Check(source != nullptr && Value(program, source->dwords[0]).op == ScalarValueOp::Phi && - DescriptorSourceResolved( - program, program.blocks[3].instructions.back().memory.resource_source), - "readlane definitions were not merged across CFG predecessors"); -} - -void TestReadLaneLoopConvergence() { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(3); - program.blocks[0].successors = {1}; - program.blocks[1].predecessors = {0, 2}; - program.blocks[1].successors = {2}; - program.blocks[2].predecessors = {1}; - program.blocks[2].successors = {1}; - program.blocks[0].instructions = {WriteLane(0, 11, 4, 0)}; - program.blocks[1].instructions = {ReadLane(4, 0, 11, 0), BufferUse(8, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto source = program.blocks[1].instructions.back().memory.resource_source; - Check(DescriptorSourceResolved(program, source), - "unchanged writelane spill did not converge across a loop backedge"); -} - -void TestReadLaneWideAndRelativeWritesInvalidateSpill() { - const auto check_invalidated = [](Opcode op, const char* message) { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(1); - Instruction overwrite; - overwrite.pc = 4; - overwrite.op = op; - overwrite.dst = Vgpr(11); - overwrite.src[0] = Imm(0); - overwrite.src_count = 1; - program.blocks[0].instructions = {WriteLane(0, 12, 4, 0), overwrite, ReadLane(8, 0, 12, 0), - BufferUse(12, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto source = program.blocks[0].instructions.back().memory.resource_source; - Check(!DescriptorSourceResolved(program, source), message); - }; - - check_invalidated(Opcode::UMadU64U32, - "64-bit vector destination left stale high-half lane provenance"); - check_invalidated(Opcode::MoveRelDestU32, - "relative vector destination left stale lane provenance"); -} - -void TestReadLaneModuloAndDynamicLane() { - Program modulo; - modulo.wave_size = 32; - modulo.user_data_count = 8; - modulo.blocks.resize(1); - modulo.blocks[0].instructions = {WriteLane(0, 11, 4, 33), ReadLane(4, 0, 11, 65), - BufferUse(8, 0)}; - std::string error; - Check(BuildScalarProvenance(modulo, &error), error.c_str()); - const auto* modulo_source = - GetDescriptorSource(modulo, modulo.blocks[0].instructions.back().memory.resource_source); - Check(modulo_source != nullptr && - Value(modulo, modulo_source->dwords[0]).op == ScalarValueOp::UserData && - Value(modulo, modulo_source->dwords[0]).imm == 4, - "readlane/writelane did not wrap the lane index to wave size"); - - Program dynamic; - dynamic.wave_size = 64; - dynamic.user_data_count = 8; - dynamic.blocks.resize(1); - auto write = WriteLane(0, 11, 4, 0); - write.src[1] = Sgpr(7); - dynamic.blocks[0].instructions = {write, ReadLane(4, 0, 11, 0), BufferUse(8, 0)}; - Check(BuildScalarProvenance(dynamic, &error), error.c_str()); - Check(!DescriptorSourceResolved(dynamic, - dynamic.blocks[0].instructions.back().memory.resource_source), - "dynamic writelane selector retained unsafe lane provenance"); -} - -void TestReadLaneEliminationSnapshotsWriteValue() { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(1); - program.blocks[0].instructions = {MoveImmediate(0, 4, 0x12345678u), WriteLane(4, 11, 4, 4), - MoveImmediate(8, 4, 0xdeadbeefu), ReadLane(12, 0, 11, 4)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto stats = EliminateReadLane(program); - Check(stats.rewritten_reads == 1 && stats.shadow_writes == 1, - "fixed readlane was not rewritten through a writelane shadow"); - const auto& instructions = program.blocks[0].instructions; - Check(instructions.size() == 5 && instructions[2].op == Opcode::MoveU32 && - instructions[2].dst.kind == OperandKind::Register && - instructions[2].dst.reg.file == RegisterFile::Scalar && - instructions[2].dst.reg.index >= 128 && instructions[2].src[0] == Sgpr(4), - "writelane did not snapshot its source into a temporary scalar"); - Check(instructions[4].op == Opcode::MoveU32 && instructions[4].dst == Sgpr(0) && - instructions[4].src[0] == instructions[2].dst, - "readlane did not consume the writelane snapshot"); -} - -void TestReadLaneEliminationMergesControlFlowWrites() { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - program.blocks[1].instructions = {WriteLane(0, 11, 4, 4)}; - program.blocks[2].instructions = {WriteLane(4, 11, 5, 4)}; - program.blocks[3].instructions = {ReadLane(8, 0, 11, 4)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto stats = EliminateReadLane(program); - Check(stats.rewritten_reads == 1 && stats.shadow_writes == 2, - "readlane merge did not shadow both reaching writelane definitions"); - const auto branch_a_temp = program.blocks[1].instructions[1].dst; - const auto branch_b_temp = program.blocks[2].instructions[1].dst; - Check(branch_a_temp == branch_b_temp && - program.blocks[3].instructions[0].op == Opcode::MoveU32 && - program.blocks[3].instructions[0].src[0] == branch_a_temp, - "control-flow writelanes did not merge through one shadow register"); -} - -void TestReadLaneEliminationRequiresWriteOnEveryPath() { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(4); - program.blocks[0].successors = {1, 2}; - program.blocks[1].predecessors = {0}; - program.blocks[1].successors = {3}; - program.blocks[2].predecessors = {0}; - program.blocks[2].successors = {3}; - program.blocks[3].predecessors = {1, 2}; - program.blocks[1].instructions = {WriteLane(0, 11, 4, 4)}; - program.blocks[3].instructions = {ReadLane(4, 0, 11, 4)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto stats = EliminateReadLane(program); - Check(stats.rewritten_reads == 0 && stats.shadow_writes == 0 && - program.blocks[3].instructions[0].op == Opcode::ReadLaneU32, - "readlane without a definition on every path was unsafely rewritten"); -} - -void TestReadLaneEliminationHonorsVectorInvalidation() { - Program program; - program.wave_size = 64; - program.user_data_count = 8; - program.blocks.resize(1); - Instruction overwrite; - overwrite.pc = 4; - overwrite.op = Opcode::MoveU32; - overwrite.dst = Vgpr(11); - overwrite.src[0] = Imm(0); - overwrite.src_count = 1; - program.blocks[0].instructions = {WriteLane(0, 11, 4, 4), overwrite, ReadLane(8, 0, 11, 4)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto stats = EliminateReadLane(program); - Check(stats.rewritten_reads == 0 && stats.shadow_writes == 0 && - program.blocks[0].instructions.back().op == Opcode::ReadLaneU32, - "readlane was rewritten across an intervening vector overwrite"); -} - -void TestReadLaneEliminationFoldsScalarLaneSelector() { - Program program; - program.wave_size = 32; - program.user_data_count = 8; - program.blocks.resize(1); - auto write = WriteLane(4, 11, 4, 0); - write.src[1] = Sgpr(7); - auto read = ReadLane(8, 0, 11, 0); - read.src[1] = Sgpr(7); - program.blocks[0].instructions = {MoveImmediate(0, 7, 33), write, read}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto stats = EliminateReadLane(program); - Check(stats.rewritten_reads == 1 && stats.shadow_writes == 1 && - program.blocks[0].instructions.back().op == Opcode::MoveU32, - "constant scalar lane selector was not folded modulo wave32"); -} - -void TestUnresolvedSourceIsMarked() { - Program program; - program.blocks.resize(1); - Instruction unknown; - unknown.op = Opcode::BitwiseNotU64; - unknown.dst = Sgpr(0); - program.blocks[0].instructions = {unknown, BufferUse(4, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto source = program.blocks[0].instructions.back().memory.resource_source; - Check(GetDescriptorSource(program, source) != nullptr && - !DescriptorSourceResolved(program, source), - "unknown descriptor tuple was incorrectly marked resolved"); -} - -void TestUnsupportedWideWriteInvalidatesBothDwords() { - Program program; - program.blocks.resize(1); - Instruction unsupported; - unsupported.op = Opcode::BitwiseNotU64; - unsupported.dst = Sgpr(0); - unsupported.src[0] = Sgpr(20); - unsupported.src_count = 1; - program.blocks[0].instructions = {Move(0, 0, 20), Move(4, 1, 21), unsupported, - BufferUse(12, 0)}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto* source = - GetDescriptorSource(program, program.blocks[0].instructions.back().memory.resource_source); - Check(source != nullptr && source->dwords[0] == ScalarProvenance::Unknown && - source->dwords[1] == ScalarProvenance::Unknown, - "unsupported wide write left a stale descriptor half"); -} - -void TestCyclicPhiIsUnresolved() { - Program program; - program.blocks.resize(2); - program.blocks[0].successors = {1}; - program.blocks[1].predecessors = {0, 1}; - program.blocks[1].successors = {1}; - Instruction increment; - increment.op = Opcode::IAddU32; - increment.dst = Sgpr(0); - increment.src[0] = Sgpr(0); - increment.src[1] = Imm(1); - increment.src_count = 2; - program.blocks[1].instructions = {BufferUse(4, 0), increment}; - - std::string error; - Check(BuildScalarProvenance(program, &error), error.c_str()); - const auto source = program.blocks[1].instructions[0].memory.resource_source; - Check(GetDescriptorSource(program, source) != nullptr && - !DescriptorSourceResolved(program, source), - "loop-carried cyclic descriptor provenance was incorrectly resolved"); - Check(BuildSrtPlan(program, &error) && program.srt.dynamic_sources.size() == 1 && - program.srt.dynamic_sources[0] == source, - "SRT planning did not isolate a cyclic descriptor as a dynamic source"); + fixture.Plan(); + Check(fixture.program.values->srt_reads.empty() && + fixture.program.values->dynamic_reads == std::vector{read}, + "dynamic scalar read received a fake flattened slot"); } void TestNestedSrtWalk() { - std::array child = {0x11111111u, 0x22334455u, 0, 0}; - std::array root = {}; - const auto child_address = reinterpret_cast(child.data()); - root[0] = static_cast(child_address); - root[1] = static_cast(child_address >> 32u); + Fixture fixture; + const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer); + const auto pointer = RawRead( + fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory); + const auto value = + RawRead(fixture, Address(fixture, pointer, Value(0u)), Value(0u), memory); + fixture.Emit(ValueOpcode::GetBufferResource, + {value, Value(0u), Value(16u), Value(0u)}); + fixture.Plan(); - Program program; - program.blocks.resize(1); - for (uint32_t i = 0; i < 2; i++) { - Instruction load; - load.pc = i * 4; - load.op = Opcode::SLoadDword; - load.dst = Sgpr(8 + i); - load.src[0] = Imm(0); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 4; - load.memory.offset = i * 4; - program.blocks[0].instructions.push_back(load); - } - Instruction child_load; - child_load.pc = 8; - child_load.op = Opcode::SLoadDword; - child_load.dst = Sgpr(16); - child_load.src[0] = Imm(0); - child_load.src_count = 1; - child_load.memory.kind = ResourceKind::ScalarBuffer; - child_load.memory.resource = 8; - child_load.memory.offset = 6; // RDNA2 SMEM ignores the low two address bits. - program.blocks[0].instructions.push_back(child_load); - program.blocks[0].instructions.push_back(BufferUse(12, 16)); - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - Check(program.srt.reads.size() == 3 && program.srt.dynamic_reads.empty(), - "nested SRT reads were not compacted and deduplicated"); - std::vector user_data(64); - SetPointer(&user_data, 4, root.data()); - std::vector flat; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(WalkSrt(program, runtime, flat, &error), error.c_str()); - Check(flat.size() == 3 && flat.back() == child[1], - "nested SRT walker read the wrong aligned dword"); + TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}}; + SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + std::vector flat; + std::string error; + Check(WalkSrt(fixture.program, runtime, flat, &error), error.c_str()); + Check(flat == std::vector({0x2000u, 0xabcdef01u}), + "nested typed SRT reads were not evaluated in dependency order"); } -void TestDynamicReadIsNotFlattened() { - std::array table = {0xaabbccddu, 0x12345678u, 0, 0}; - Program program; - program.blocks.resize(1); - Instruction load; - load.op = Opcode::SLoadDword; - load.dst = Sgpr(16); - load.src[0] = Sgpr(6); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 4; - program.blocks[0].instructions = {load, BufferUse(4, 16)}; +void TestShaderBaseAndUserData() { + Fixture fixture; + const auto base = fixture.Emit(ValueOpcode::GetShaderBase); + const auto pair = fixture.Emit(ValueOpcode::UnpackUint2x32, {base}); + const auto low = + fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(0u)}); + const auto high = + fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(1u)}); + const auto user = fixture.Emit(ValueOpcode::GetUserData, + {Value(static_cast(2))}); + const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)}); + fixture.Plan(); + fixture.program.values->descriptor_sources.push_back( + {.dwords = {low, high, sum}, .dword_count = 3}); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - Check(program.srt.reads.empty() && program.srt.dynamic_reads.size() == 1, - "dynamic ReadConst was assigned a flat-buffer offset"); - std::vector user_data(64); - SetPointer(&user_data, 4, table.data()); - user_data[6] = 4; - DescriptorValue descriptor; - const auto source = program.blocks[0].instructions[1].memory.resource_source; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(EvaluateDescriptorSource(program, source, 4, runtime, descriptor, &error), error.c_str()); - Check(descriptor.dwords[0] == table[1], "dynamic ReadConst evaluated the wrong dword"); + const std::array user_data{0x20u}; + SrtRuntime runtime{.user_data = user_data, + .shader_base = 0x12345678abcdef00ull}; + DescriptorValue result; + std::string error; + Check(EvaluateDescriptorSource(fixture.program, 0, 0x90, runtime, result, + &error), + error.c_str()); + Check(result.dword_count == 3 && result.dwords[0] == 0xabcdef00u && + result.dwords[1] == 0x12345678u && result.dwords[2] == 0x24u, + "shader-relative typed descriptor expression evaluated incorrectly"); } -void TestReadConstBufferWalk() { - std::array table = {0x76543210u, 0, 0, 0}; - Program program; - program.blocks.resize(1); - Instruction load; - load.op = Opcode::SBufferLoadDword; - load.dst = Sgpr(16); - load.src[0] = Imm(0); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 1; - program.blocks[0].instructions = {load, BufferUse(4, 16)}; +void TestCarryAndBitFields() { + Fixture fixture; + const auto carry = + fixture.Emit(ValueOpcode::IAddCarry32, {Value(0xffffffffu), Value(2u)}); + const auto low = + fixture.Emit(ValueOpcode::CompositeExtractU32x2, {carry, Value(0u)}); + const auto high = + fixture.Emit(ValueOpcode::CompositeExtractU32x2, {carry, Value(1u)}); + const auto inserted = + fixture.Emit(ValueOpcode::BitFieldInsert, + {Value(0u), Value(0x89abcdefu), Value(0u), Value(32u)}); + const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract, + {Value(0x000000f0u), Value(4u), Value(4u)}); + fixture.Plan(); + fixture.program.values->descriptor_sources.push_back( + {.dwords = {low, high, inserted, sign}, .dword_count = 4}); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - std::vector user_data(64); - SetPointer(&user_data, 4, table.data()); - user_data[6] = 4; - std::vector flat; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(WalkSrt(program, runtime, flat, &error), error.c_str()); - Check(flat.size() == 1 && flat[0] == table[0], "ReadConstBuffer walker result was wrong"); + DescriptorValue result; + std::string error; + Check(EvaluateDescriptorSource(fixture.program, 0, 0, {}, result, &error), + error.c_str()); + Check(result.dwords[0] == 1u && result.dwords[1] == 1u && + result.dwords[2] == 0x89abcdefu && result.dwords[3] == 0xffffffffu, + "typed carry or bit-field runtime evaluation is incorrect"); } -void TestM0ScalarOffset() { - std::array table = {0x01234567u, 0x89abcdefu}; - Program program; - program.blocks.resize(1); - Instruction set_m0; - set_m0.op = Opcode::MoveU32; - set_m0.dst = M0(); - set_m0.src[0] = Imm(4); - set_m0.src_count = 1; - Instruction load; - load.pc = 4; - load.op = Opcode::SLoadDword; - load.dst = Sgpr(16); - load.src[0] = M0(); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 4; - program.blocks[0].instructions = {set_m0, load, BufferUse(8, 16)}; +void TestInvariantAndDivergentPhi() { + Fixture fixture(3); + auto &invariant = fixture.BlockAt(2).AppendNewInst(ValueOpcode::Phi); + invariant.SetFlags(Type::U32); + invariant.AddPhiOperand(&fixture.BlockAt(0), Value(7u)); + invariant.AddPhiOperand(&fixture.BlockAt(1), Value(7u)); + auto &divergent = fixture.BlockAt(2).AppendNewInst(ValueOpcode::Phi); + divergent.SetFlags(Type::U32); + divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u)); + divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u)); + fixture.Plan(); + fixture.program.values->descriptor_sources.push_back( + {.dwords = {Value(&invariant)}, .dword_count = 1}); + fixture.program.values->descriptor_sources.push_back( + {.dwords = {Value(&divergent)}, .dword_count = 1}); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - Check(program.srt.reads.size() == 1 && program.srt.dynamic_reads.empty(), - "constant M0 offset was not flattened"); - std::vector user_data(64); - SetPointer(&user_data, 4, table.data()); - std::vector flat; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(WalkSrt(program, runtime, flat, &error), error.c_str()); - Check(flat.size() == 1 && flat[0] == table[1], "M0 scalar offset evaluated incorrectly"); + DescriptorValue result; + std::string error; + Check( + EvaluateDescriptorSource(fixture.program, 0, 0x100, {}, result, &error) && + result.dwords[0] == 7u, + "loop-invariant typed phi was rejected"); + result.dword_count = 4; + result.dwords[0] = 0xdeadbeefu; + Check(!EvaluateDescriptorSource(fixture.program, 1, 0x104, {}, result, + &error) && + result.dword_count == 4 && result.dwords[0] == 0xdeadbeefu && + error.find("runtime-dependent") != std::string::npos, + "divergent phi did not fail transactionally"); } -void TestM0ArithmeticPreservesScc() { - Program program; - program.blocks.resize(1); - Instruction low; - low.op = Opcode::ScalarAddCarryU32; - low.dst = M0(); - low.src[0] = Sgpr(0); - low.src[1] = Imm(1); - low.src[2] = Imm(0); - low.src_count = 3; - Instruction high = low; - high.pc = 4; - high.dst = Sgpr(16); - high.src[0] = Imm(5); - high.src[1] = Imm(0); - high.src[2].kind = OperandKind::Register; - high.src[2].reg.file = RegisterFile::Scc; - program.blocks[0].instructions = {low, - high, - MoveImmediate(8, 17, 0), - MoveImmediate(12, 18, 0), - MoveImmediate(16, 19, 0), - BufferUse(20, 16)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const std::array user_data = {0xffffffffu}; - const auto source = program.blocks[0].instructions.back().memory.resource_source; - DescriptorValue descriptor; - const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; - Check(EvaluateDescriptorSource(program, source, 20, runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[0] == 6, "arithmetic M0 destination lost its SCC carry side effect"); +void TestControlDependentStandaloneLoadStaysTyped() { + Fixture fixture(3); + const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer); + auto &base = fixture.BlockAt(2).AppendNewInst(ValueOpcode::Phi); + base.SetFlags(Type::U32); + base.AddPhiOperand(&fixture.BlockAt(0), Value(0x1000u)); + base.AddPhiOperand(&fixture.BlockAt(1), Value(0x2000u)); + const auto read = + RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u), + memory, 2); + fixture.Plan(); + Check(fixture.program.values->srt_reads.empty() && + read.ResolveInstruction()->GetOpcode() == + ValueOpcode::LoadAddressU32 && + !fixture.program.values->memory_info[memory].planning_only, + "control-dependent standalone scalar load was flattened into a host " + "snapshot"); } -void TestShaderRelativePointer() { - std::array table = {}; - Program program; - program.blocks.resize(1); - Instruction getpc_low = MoveImmediate(0x20, 0, 0x24); - getpc_low.src[0].kind = OperandKind::PcRelativeU32; - Instruction getpc_high = MoveImmediate(0x20, 1, 0); - program.blocks[0].instructions = {getpc_low, getpc_high, BufferUse(0x28, 0)}; +void TestRuntime64BitDescriptorOps() { + Fixture fixture; + const auto shifted = fixture.Emit(ValueOpcode::ShiftLeftLogical64, + {Value(uint64_t{0x1234u}), Value(32u)}); + const auto masked = + fixture.Emit(ValueOpcode::BitwiseAnd64, + {shifted, Value(uint64_t{0x0000ffff00000000ull})}); + const auto combined = fixture.Emit(ValueOpcode::BitwiseOr64, + {masked, Value(uint64_t{0xabcdu})}); + const auto pair = fixture.Emit(ValueOpcode::UnpackUint2x32, {combined}); + const auto low = + fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(0u)}); + const auto high = + fixture.Emit(ValueOpcode::CompositeExtractU32x2, {pair, Value(1u)}); + fixture.Plan(); + fixture.program.values->descriptor_sources.push_back( + {.dwords = {low, high}, .dword_count = 2}); - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const auto source = program.blocks[0].instructions.back().memory.resource_source; - DescriptorValue descriptor; - const auto shader_base = reinterpret_cast(table.data()) - 0x24; - const std::array user_data = {}; - const SrtRuntime runtime {user_data, shader_base, nullptr, nullptr}; - Check(EvaluateDescriptorSource(program, source, 0x28, runtime, descriptor, &error), - error.c_str()); - const auto expected = reinterpret_cast(table.data()); - Check(descriptor.dwords[0] == static_cast(expected) && - descriptor.dwords[1] == static_cast(expected >> 32u), - "shader-relative pointer lost its high half or carry"); + DescriptorValue result; + std::string error; + Check(EvaluateDescriptorSource(fixture.program, 0, 0, {}, result, &error) && + result.dwords[0] == 0xabcdu && result.dwords[1] == 0x1234u, + "64-bit typed descriptor shift/mask evaluation is incorrect"); } -void TestSmemComponentAlignment() { - std::array table = {0x11111111u, 0x22222222u}; - std::vector user_data(20); - SetPointer(&user_data, 4, table.data()); +void TestConstantBufferBounds() { + Fixture fixture; + const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer); + const auto buffer = + fixture.Emit(ValueOpcode::GetBufferResource, + {Value(0x3000u), Value(0u), Value(16u), Value(0u)}); + const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer, + {buffer, Value(12u)}, memory); + fixture.Emit(ValueOpcode::GetBufferResource, + {read, Value(0u), Value(16u), Value(0u)}); + fixture.Plan(); - Program raw; - raw.blocks.resize(1); - Instruction raw_load; - raw_load.pc = 4; - raw_load.op = Opcode::SLoadDword; - raw_load.dst = Sgpr(16); - raw_load.src[0] = Imm(2); - raw_load.src_count = 1; - raw_load.memory.kind = ResourceKind::ScalarBuffer; - raw_load.memory.resource = 4; - raw_load.memory.offset = 2; - raw.blocks[0].instructions = {raw_load, BufferUse(12, 16)}; - std::string error; - Check(BuildScalarProvenance(raw, &error) && BuildSrtPlan(raw, &error), error.c_str()); - std::vector flat; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(WalkSrt(raw, runtime, flat, &error), error.c_str()); - Check(flat.size() == 1 && flat[0] == table[0], - "raw S_LOAD did not clear each address component's low bits"); + TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}}; + SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image}; + std::vector flat; + std::string error; + Check(WalkSrt(fixture.program, runtime, flat, &error) && + flat == std::vector{0xa5a5a5a5u}, + error.c_str()); - Program buffer; - buffer.blocks.resize(1); - Instruction buffer_load = raw_load; - buffer_load.op = Opcode::SBufferLoadDword; - buffer_load.memory.resource = 1; - buffer.blocks[0].instructions = {buffer_load, BufferUse(12, 16)}; - user_data[6] = sizeof(table); - Check(BuildScalarProvenance(buffer, &error) && BuildSrtPlan(buffer, &error), error.c_str()); - Check(WalkSrt(buffer, runtime, flat, &error), error.c_str()); - Check(flat.size() == 1 && flat[0] == table[1], - "S_BUFFER_LOAD did not align the summed final address"); - - buffer.blocks[0].instructions[0].src[0] = Imm(0); - user_data[4] += 2; - Check(BuildScalarProvenance(buffer, &error) && BuildSrtPlan(buffer, &error), error.c_str()); - Check(WalkSrt(buffer, runtime, flat, &error), error.c_str()); - Check(flat.size() == 1 && flat[0] == table[0], - "S_BUFFER_LOAD did not clear the descriptor base low bits before " - "addition"); + Fixture overflow; + const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer); + const auto overflow_buffer = + overflow.Emit(ValueOpcode::GetBufferResource, + {Value(0x3000u), Value(0u), Value(16u), Value(0u)}); + const auto overflow_read = + overflow.EmitMemory(ValueOpcode::ReadConstBuffer, + {overflow_buffer, Value(16u)}, overflow_memory); + overflow.Emit(ValueOpcode::GetBufferResource, + {overflow_read, Value(0u), Value(16u), Value(0u)}); + overflow.Plan(); + flat = {0x55u}; + Check(!WalkSrt(overflow.program, runtime, flat, &error) && + flat == std::vector{0x55u} && + error.find("exceeds size") != std::string::npos, + "out-of-bounds constant-buffer walk was not transactional"); } -void TestReadConstSignedAddress() { - std::array table = {0x11111111u, 0x22222222u}; - std::vector user_data(24); - SetPointer(&user_data, 4, table.data() + 1); +void TestReadLaneElimination() { + Fixture fixture; + const auto undef = fixture.Emit(ValueOpcode::UndefU32); + const auto write = fixture.Emit(ValueOpcode::WriteLane, + {undef, Value(0xdeadbeefu), Value(5u)}); + const auto read = fixture.Emit(ValueOpcode::ReadLane, {write, Value(5u)}); + const auto use = fixture.Emit(ValueOpcode::IAdd32, {read, Value(1u)}); + const auto stats = EliminateReadLane(*fixture.program.values, 64); + Check(stats.rewritten_reads == 1 && + use.ResolveInstruction()->Arg(0).Resolve() == Value(0xdeadbeefu), + "fixed-lane typed read was not rewritten from its SSA write chain"); - Program program; - program.blocks.resize(1); - Instruction load; - load.pc = 4; - load.op = Opcode::SLoadDword; - load.dst = Sgpr(16); - load.src[0] = Sgpr(20); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 4; - load.memory.offset = static_cast(-8); - program.blocks[0].instructions = {MoveImmediate(0, 20, 4), load, BufferUse(8, 16)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - std::vector flat; - const SrtRuntime runtime {user_data, 0, ReadHostMemory, nullptr}; - Check(WalkSrt(program, runtime, flat, &error), error.c_str()); - Check(flat.size() == 1 && flat[0] == table[0], - "signed S_LOAD offset resolved the wrong SBASE-relative address"); - - user_data[4] = 0; - user_data[5] = 0; - Check(!WalkSrt(program, runtime, flat, &error) && - error.find("outside the 48-bit address space") != std::string::npos, - "signed S_LOAD address underflow did not fail closed"); + const auto selector = fixture.Emit(ValueOpcode::GetUserData, + {Value(static_cast(2))}); + const auto dynamic = fixture.Emit(ValueOpcode::ReadLane, {write, selector}); + fixture.Emit(ValueOpcode::IAdd32, {dynamic, Value(1u)}); + Check(EliminateReadLane(*fixture.program.values, 64).rewritten_reads == 0, + "dynamic-lane read was rewritten unsafely"); } -void TestReadConstBufferBounds() { - std::array table = {0x11111111u, 0x22222222u}; - Program program; - program.stage = ShaderType::Compute; - program.shader_hash = 0x1234; - program.blocks.resize(1); - Instruction load; - load.pc = 0x40; - load.op = Opcode::SBufferLoadDword; - load.dst = Sgpr(16); - load.src[0] = Imm(0); - load.src_count = 1; - load.memory.kind = ResourceKind::ScalarBuffer; - load.memory.resource = 1; - load.memory.offset = sizeof(table); - program.blocks[0].instructions = {load, BufferUse(0x44, 16)}; - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - std::vector user_data(20); - SetPointer(&user_data, 4, table.data()); - user_data[5] |= 4u << 16u; - user_data[6] = 2; - ReadCounter counter; - std::vector flat; - const SrtRuntime runtime {user_data, 0, CountedHostMemory, &counter}; - Check(!WalkSrt(program, runtime, flat, &error), - "out-of-range S_BUFFER_LOAD unexpectedly succeeded"); - Check(counter.reads == 0 && error.find("exceeds size=8") != std::string::npos && - error.find("hash=0x0000000000001234") != std::string::npos && - error.find("stage=compute") != std::string::npos && - error.find("pc=0x00000044") != std::string::npos, - "S_BUFFER_LOAD bounds failure read memory or lost shader context"); +void TestOptimizationPipeline() { + Fixture fixture; + const auto sum = fixture.Emit(ValueOpcode::IAdd32, {Value(40u), Value(2u)}); + const auto kept = fixture.Emit(ValueOpcode::BitwiseOr32, {sum, Value(0u)}); + fixture.Emit(ValueOpcode::ReferenceU32, {kept}); + fixture.Emit(ValueOpcode::IMul32, {Value(6u), Value(7u)}); + + ConstantPropagationPass(fixture.program.values->blocks); + RemoveIdentities(fixture.program.values->blocks); + EliminateDeadCode(fixture.program.values->blocks); + + const auto &instructions = fixture.BlockAt().Instructions(); + Check(instructions.size() == 1 && + instructions.front().GetOpcode() == ValueOpcode::ReferenceU32 && + instructions.front().Arg(0).Resolve() == Value(42u), + "typed constant propagation, identity folding, or dead-code " + "elimination regressed"); } -void TestSubBorrowPointerChain() { - Program program; - program.blocks.resize(1); - Instruction low; - low.op = Opcode::ScalarSubBorrowU32; - low.dst = Sgpr(16); - low.src[0] = Sgpr(0); - low.src[1] = Imm(4); - low.src_count = 2; - Instruction high; - high.pc = 4; - high.op = Opcode::ScalarSubBorrowCarryU32; - high.dst = Sgpr(17); - high.src[0] = Sgpr(1); - high.src[1] = Imm(0); - high.src[2].kind = OperandKind::Register; - high.src[2].reg.file = RegisterFile::Scc; - high.src_count = 3; - program.blocks[0].instructions = {low, high, MoveImmediate(8, 18, 0), MoveImmediate(12, 19, 0), - BufferUse(16, 16)}; - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const std::array user_data = {2, 5}; - DescriptorValue descriptor; - const auto source = program.blocks[0].instructions.back().memory.resource_source; - const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; - Check(EvaluateDescriptorSource(program, source, 16, runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[0] == 0xfffffffeu && descriptor.dwords[1] == 4, - "scalar subtraction/borrow pointer chain evaluated incorrectly"); -} - -void TestCommonScalarPointerOps() { - Program program; - program.blocks.resize(1); - const auto binary = [](Opcode op, uint32_t dst, Operand lhs, Operand rhs) { - Instruction inst; - inst.op = op; - inst.dst = Sgpr(dst); - inst.src[0] = lhs; - inst.src[1] = rhs; - inst.src_count = 2; - return inst; - }; - Instruction add3; - add3.op = Opcode::IAdd3U32; - add3.dst = Sgpr(19); - add3.src[0] = Sgpr(3); - add3.src[1] = Imm(1); - add3.src[2] = Imm(2); - add3.src_count = 3; - program.blocks[0].instructions = {binary(Opcode::BitwiseXorU32, 16, Sgpr(0), Sgpr(1)), - binary(Opcode::BitwiseAndNotU32, 17, Sgpr(0), Sgpr(1)), - binary(Opcode::IMulU32, 18, Sgpr(2), Imm(4)), add3, - BufferUse(16, 16)}; - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - const std::array user_data = {0xf0f0u, 0x0ff0u, 7u, 9u}; - DescriptorValue descriptor; - const auto source = program.blocks[0].instructions.back().memory.resource_source; - const SrtRuntime runtime {user_data, 0, nullptr, nullptr}; - Check(EvaluateDescriptorSource(program, source, 16, runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[0] == (user_data[0] ^ user_data[1]) && - descriptor.dwords[1] == (user_data[0] & ~user_data[1]) && - descriptor.dwords[2] == 28 && descriptor.dwords[3] == 12, - "common scalar pointer operations evaluated incorrectly"); -} - -void TestBitFieldMaskDescriptor() { - Program program; - program.blocks.resize(1); - auto mask = MoveImmediate(4, 29, 0); - mask.op = Opcode::BitFieldMaskU32; - mask.src[0] = Imm(12); - mask.src[1] = Imm(12); - mask.src_count = 2; - auto high = MoveImmediate(8, 30, 0x05500000u); - high.op = Opcode::MoveU64; - program.blocks[0].instructions = {MoveImmediate(0, 28, 0x92u), mask, high, BufferUse(12, 28)}; - - std::string error; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - DescriptorValue descriptor; - const SrtRuntime runtime {{}, 0, nullptr, nullptr}; - Check(EvaluateDescriptorSource(program, - program.blocks[0].instructions.back().memory.resource_source, 16, - runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[0] == 0x92u && descriptor.dwords[1] == 0x00fff000u && - descriptor.dwords[2] == 0x05500000u && descriptor.dwords[3] == 0, - "production sampler bit-field mask evaluated incorrectly"); - - mask.src[0] = Imm(0); - program.blocks[0].instructions = {MoveImmediate(0, 28, 0x92u), mask, high, BufferUse(12, 28)}; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - Check(EvaluateDescriptorSource(program, - program.blocks[0].instructions.back().memory.resource_source, 12, - runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[1] == 0, "zero-width bit-field mask was not zero"); - - mask.src[0] = Imm(31); - mask.src[1] = Imm(31); - program.blocks[0].instructions = {MoveImmediate(0, 28, 0x92u), mask, high, BufferUse(12, 28)}; - Check(BuildScalarProvenance(program, &error) && BuildSrtPlan(program, &error), error.c_str()); - Check(EvaluateDescriptorSource(program, - program.blocks[0].instructions.back().memory.resource_source, 12, - runtime, descriptor, &error), - error.c_str()); - Check(descriptor.dwords[1] == 0x80000000u, - "maximum bit-field mask count/offset evaluated incorrectly"); +void TestUndefinedRuntimeValueFails() { + Fixture fixture; + const auto undef = fixture.Emit(ValueOpcode::UndefU32); + fixture.Plan(); + fixture.program.values->descriptor_sources.push_back( + {.dwords = {undef}, .dword_count = 1}); + DescriptorValue result; + result.dword_count = 3; + std::string error; + Check(!EvaluateDescriptorSource(fixture.program, 0, 0x200, {}, result, + &error) && + result.dword_count == 3 && + error.find("undefined typed runtime value") != std::string::npos, + "undefined typed descriptor source did not fail transactionally"); } } // namespace +namespace Common { + +int DbgExitIfHandler(const char *, const char *, int) { return 1; } + +void DbgExit(int) { std::abort(); } + +} // namespace Common + int main() { - try { - TestPerUseDescriptorDefinitions(); - TestCfgPhi(); - TestNestedLoopPhiConvergence(); - TestDiamondReadPathsAreDynamic(); - TestEquivalentConstantPhiIsStatic(); - TestWideMoveInvalidatesAndCopiesBothDwords(); - TestScalarCarryChain(); - TestReadConstDefinition(); - TestReadConstBufferAndValueNumbering(); - TestReadLaneDescriptorSpill(); - TestReadLaneVectorOverwriteInvalidatesSpill(); - TestReadLanePhi(); - TestReadLaneLoopConvergence(); - TestReadLaneWideAndRelativeWritesInvalidateSpill(); - TestReadLaneModuloAndDynamicLane(); - TestReadLaneEliminationSnapshotsWriteValue(); - TestReadLaneEliminationMergesControlFlowWrites(); - TestReadLaneEliminationRequiresWriteOnEveryPath(); - TestReadLaneEliminationHonorsVectorInvalidation(); - TestReadLaneEliminationFoldsScalarLaneSelector(); - TestUnresolvedSourceIsMarked(); - TestUnsupportedWideWriteInvalidatesBothDwords(); - TestCyclicPhiIsUnresolved(); - TestNestedSrtWalk(); - TestDynamicReadIsNotFlattened(); - TestReadConstBufferWalk(); - TestM0ScalarOffset(); - TestM0ArithmeticPreservesScc(); - TestShaderRelativePointer(); - TestSmemComponentAlignment(); - TestReadConstSignedAddress(); - TestReadConstBufferBounds(); - TestSubBorrowPointerChain(); - TestCommonScalarPointerOps(); - TestBitFieldMaskDescriptor(); - std::cout << "ScalarProvenanceTests: all cases passed\n"; - return 0; - } catch (const std::exception& e) { - std::cerr << "ScalarProvenanceTests: failed: " << e.what() << '\n'; - return 1; - } + try { + TestImmediateFlatteningAndGvn(); + TestDynamicReadRemainsTyped(); + TestNestedSrtWalk(); + TestShaderBaseAndUserData(); + TestCarryAndBitFields(); + TestInvariantAndDivergentPhi(); + TestControlDependentStandaloneLoadStaysTyped(); + TestRuntime64BitDescriptorOps(); + TestConstantBufferBounds(); + TestReadLaneElimination(); + TestOptimizationPipeline(); + TestUndefinedRuntimeValueFails(); + std::cout << "TypedValuePlanningTests: all cases passed\n"; + return 0; + } catch (const std::exception &e) { + std::cerr << "TypedValuePlanningTests: failed: " << e.what() << '\n'; + return 1; + } } + +// Keep this focused standalone target self-contained by amalgamating its small +// typed-IR implementation set. +#include "../src/graphics/shader/recompiler/ir/Block.cpp" +#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp" +#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp" +#include "../src/graphics/shader/recompiler/ir/Type.cpp" +#include "../src/graphics/shader/recompiler/ir/Value.cpp" +#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" +#include "../src/graphics/shader/recompiler/ir/ValueProgram.cpp" diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 5f515847d..00b55a02a 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -35,10 +35,11 @@ #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/presentation/window/windowInternal.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" -#include "graphics/shader/recompiler/emitter/SpirvBuilder.h" -#include "graphics/shader/recompiler/emitter/SpirvEmitter.h" -#include "graphics/shader/recompiler/ir/BindingLayout.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" +#include "graphics/shader/recompiler/ir/passes/BindingLayout.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/rectListShader.h" #include "graphics/shader/shader.h" #include "kernel/memory.h" @@ -93,23 +94,28 @@ namespace Libs::Graphics { template -concept HasGetDownloadBuffer = requires(Cache& cache) { cache.GetDownloadBuffer(uint64_t {1}); }; +concept HasGetDownloadBuffer = + requires(Cache &cache) { cache.GetDownloadBuffer(uint64_t{1}); }; static_assert(!HasGetDownloadBuffer); template -concept HasSynchronizeImageToBuffer = - requires(Cache& cache) { cache.SynchronizeImageToBuffer(uint64_t {1}, uint64_t {1}); }; +concept HasSynchronizeImageToBuffer = requires(Cache &cache) { + cache.SynchronizeImageToBuffer(uint64_t{1}, uint64_t{1}); +}; template -concept HasObtainBufferForImageCopy = - requires(Cache& cache) { cache.ObtainBufferForImageCopy(uint64_t {1}, uint64_t {1}); }; +concept HasObtainBufferForImageCopy = requires(Cache &cache) { + cache.ObtainBufferForImageCopy(uint64_t{1}, uint64_t{1}); +}; template -concept HasObtainBufferForImageWrite = - requires(Cache& cache) { cache.ObtainBufferForImageWrite(uint64_t {1}, uint64_t {1}); }; +concept HasObtainBufferForImageWrite = requires(Cache &cache) { + cache.ObtainBufferForImageWrite(uint64_t{1}, uint64_t{1}); +}; template -concept HasDiscardGpuDirtyBytes = - requires(Cache& cache) { cache.DiscardGpuDirtyBytes(uint64_t {1}, uint64_t {1}); }; +concept HasDiscardGpuDirtyBytes = requires(Cache &cache) { + cache.DiscardGpuDirtyBytes(uint64_t{1}, uint64_t{1}); +}; template -concept HasGpuOwnedImageSource = requires(Source& source) { source.gpu_owned; }; +concept HasGpuOwnedImageSource = requires(Source &source) { source.gpu_owned; }; static_assert(!HasSynchronizeImageToBuffer); static_assert(!HasObtainBufferForImageCopy); static_assert(!HasObtainBufferForImageWrite); @@ -117,7 +123,7 @@ static_assert(!HasDiscardGpuDirtyBytes); static_assert(!HasGpuOwnedImageSource); template -concept HasLegacyImageLayout = requires(Backing& backing) { backing.layout; }; +concept HasLegacyImageLayout = requires(Backing &backing) { backing.layout; }; static_assert(!HasLegacyImageLayout); template @@ -126,18761 +132,20140 @@ template concept HasPolicyTileImageBacking = requires { &Manager::TileImageBacking; }; static_assert(!HasPolicyDetileImage); static_assert(!HasPolicyTileImageBacking); -static_assert(BlitHelper::ColorToMsDepthLayout == vk::ImageLayout::eDepthStencilAttachmentOptimal); +static_assert(BlitHelper::ColorToMsDepthLayout == + vk::ImageLayout::eDepthStencilAttachmentOptimal); struct BufferCacheTestAccess { - static void SetGarbageCollectionThresholds(BufferCache& cache, uint64_t trigger, - uint64_t critical) { - cache.m_trigger_gc_memory = trigger; - cache.m_critical_gc_memory = critical; - } + static void SetGarbageCollectionThresholds(BufferCache &cache, + uint64_t trigger, + uint64_t critical) { + cache.m_trigger_gc_memory = trigger; + cache.m_critical_gc_memory = critical; + } - static StreamBuffer& DownloadBuffer(BufferCache& cache) { return cache.m_download_buffer; } + static StreamBuffer &DownloadBuffer(BufferCache &cache) { + return cache.m_download_buffer; + } - static bool SynchronizeBufferFromImage(BufferCache& cache, Buffer& buffer, uint64_t address, - uint64_t size) { - return cache.SynchronizeBufferFromImage(buffer, address, size); - } + static bool SynchronizeBufferFromImage(BufferCache &cache, Buffer &buffer, + uint64_t address, uint64_t size) { + return cache.SynchronizeBufferFromImage(buffer, address, size); + } }; struct StreamBufferTestAccess { - static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size, - uint64_t& alignment) { - return StreamBuffer::NormalizeReservation(coherent, atom, size, alignment); - } + static bool NormalizeReservation(bool coherent, uint64_t atom, uint64_t &size, + uint64_t &alignment) { + return StreamBuffer::NormalizeReservation(coherent, atom, size, alignment); + } }; struct ImageTestAccess { - static uint32_t CopyRows(uint64_t row_size, uint32_t rows, uint64_t capacity) { - return Image::CopyRows(row_size, rows, capacity); - } + static uint32_t CopyRows(uint64_t row_size, uint32_t rows, + uint64_t capacity) { + return Image::CopyRows(row_size, rows, capacity); + } }; struct TileManagerTestAccess { - static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride, uint64_t active, - uint32_t remaining, uint64_t alignment, uint64_t max_range, - uint32_t max_groups) { - return TileManager::ConversionRows(offset, row_stride, active, remaining, alignment, - max_range, max_groups); - } + static uint32_t ConversionRows(uint64_t offset, uint64_t row_stride, + uint64_t active, uint32_t remaining, + uint64_t alignment, uint64_t max_range, + uint32_t max_groups) { + return TileManager::ConversionRows(offset, row_stride, active, remaining, + alignment, max_range, max_groups); + } }; struct TextureCacheTestAccess { - static_assert(TextureCache::ImagePageTable::kPageBits == 20); - static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 40); - static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 10); + static_assert(TextureCache::ImagePageTable::kPageBits == 20); + static_assert(TextureCache::ImagePageTable::kAddressSpaceBits == 40); + static_assert(TextureCache::ImagePageTable::kFirstLevelBits == 10); - static std::unique_lock Lock(TextureCache& cache) { - return std::unique_lock(cache.m_lock); - } + static std::unique_lock Lock(TextureCache &cache) { + return std::unique_lock(cache.m_lock); + } - static void ConfigureGarbageCollection(TextureCache& cache, std::span oldest, - uint64_t tick, uint64_t pressure) { - cache.m_trigger_gc_memory = 0; - cache.m_pressure_gc_memory = pressure; - cache.m_critical_gc_memory = UINT64_MAX; - cache.m_gc_tick = tick; - std::vector live; - cache.m_lru_cache = {}; - for (auto& slot: cache.m_slots) { - if (slot.image != nullptr && slot.image->registered) { - slot.image->tick_accessed_last = cache.m_scheduler.CurrentTick(); - live.push_back( - {static_cast(&slot - cache.m_slots.data()), slot.generation}); - } - } - for (const auto id: oldest) { - const auto owner = cache.ResolveOwner(id); - if (owner != nullptr && owner->registered) { - owner->tick_accessed_last = 0; - owner->lru_id = cache.m_lru_cache.Insert(id, 0); - } - } - for (const auto id: live) { - if (std::ranges::find(oldest, id) == oldest.end()) { - cache.ResolveImage(id).lru_id = cache.m_lru_cache.Insert(id, tick); - } - } - } + static void ConfigureGarbageCollection(TextureCache &cache, + std::span oldest, + uint64_t tick, uint64_t pressure) { + cache.m_trigger_gc_memory = 0; + cache.m_pressure_gc_memory = pressure; + cache.m_critical_gc_memory = UINT64_MAX; + cache.m_gc_tick = tick; + std::vector live; + cache.m_lru_cache = {}; + for (auto &slot : cache.m_slots) { + if (slot.image != nullptr && slot.image->registered) { + slot.image->tick_accessed_last = cache.m_scheduler.CurrentTick(); + live.push_back({static_cast(&slot - cache.m_slots.data()), + slot.generation}); + } + } + for (const auto id : oldest) { + const auto owner = cache.ResolveOwner(id); + if (owner != nullptr && owner->registered) { + owner->tick_accessed_last = 0; + owner->lru_id = cache.m_lru_cache.Insert(id, 0); + } + } + for (const auto id : live) { + if (std::ranges::find(oldest, id) == oldest.end()) { + cache.ResolveImage(id).lru_id = cache.m_lru_cache.Insert(id, tick); + } + } + } - static bool Contains(const TextureCache& cache, ImageId id) { - const auto owner = cache.ResolveOwner(id); - return owner != nullptr && owner->registered; - } + static bool Contains(const TextureCache &cache, ImageId id) { + const auto owner = cache.ResolveOwner(id); + return owner != nullptr && owner->registered; + } - static std::vector FindImages(TextureCache& cache, uint64_t address, uint64_t size, - bool page_overlap) { - std::lock_guard lock(cache.m_lock); - const auto found = cache.FindImagesInRegion(address, size, page_overlap); - std::vector result; - result.reserve(found.size()); - for (const auto id: found) { - result.push_back(id); - } - return result; - } + static std::vector FindImages(TextureCache &cache, uint64_t address, + uint64_t size, bool page_overlap) { + std::lock_guard lock(cache.m_lock); + const auto found = cache.FindImagesInRegion(address, size, page_overlap); + std::vector result; + result.reserve(found.size()); + for (const auto id : found) { + result.push_back(id); + } + return result; + } - static size_t PageOwnerCount(TextureCache& cache, uint64_t address) { - std::lock_guard lock(cache.m_lock); - const auto* owners = cache.m_image_page_table.Find( - static_cast(address >> TextureCache::ImagePageTable::kPageBits)); - return owners == nullptr ? 0 : owners->size(); - } + static size_t PageOwnerCount(TextureCache &cache, uint64_t address) { + std::lock_guard lock(cache.m_lock); + const auto *owners = cache.m_image_page_table.Find(static_cast( + address >> TextureCache::ImagePageTable::kPageBits)); + return owners == nullptr ? 0 : owners->size(); + } - static size_t OwnedPageCount(TextureCache& cache, uint64_t address, uint64_t size, ImageId id) { - std::lock_guard lock(cache.m_lock); - TextureCache::ImagePageTable::PageRange pages {}; - if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) { - return 0; - } - size_t count = 0; - for (size_t page = pages.first; page < pages.last_exclusive; ++page) { - const auto* owners = cache.m_image_page_table.Find(page); - count += owners != nullptr && owners->Contains(id) ? 1 : 0; - } - return count; - } + static size_t OwnedPageCount(TextureCache &cache, uint64_t address, + uint64_t size, ImageId id) { + std::lock_guard lock(cache.m_lock); + TextureCache::ImagePageTable::PageRange pages{}; + if (!TextureCache::ImagePageTable::TryGetPageRange(address, size, pages)) { + return 0; + } + size_t count = 0; + for (size_t page = pages.first; page < pages.last_exclusive; ++page) { + const auto *owners = cache.m_image_page_table.Find(page); + count += owners != nullptr && owners->Contains(id) ? 1 : 0; + } + return count; + } - static void AddPageOwner(TextureCache& cache, uint64_t address, ImageId id) { - std::lock_guard lock(cache.m_lock); - cache.m_image_page_table[static_cast( - address >> TextureCache::ImagePageTable::kPageBits)] - .push_back(id); - } + static void AddPageOwner(TextureCache &cache, uint64_t address, ImageId id) { + std::lock_guard lock(cache.m_lock); + cache + .m_image_page_table[static_cast( + address >> TextureCache::ImagePageTable::kPageBits)] + .push_back(id); + } - static bool RemovePageOwner(TextureCache& cache, uint64_t address, ImageId id) { - std::lock_guard lock(cache.m_lock); - auto* owners = cache.m_image_page_table.Find( - static_cast(address >> TextureCache::ImagePageTable::kPageBits)); - return owners != nullptr && owners->Erase(id); - } + static bool RemovePageOwner(TextureCache &cache, uint64_t address, + ImageId id) { + std::lock_guard lock(cache.m_lock); + auto *owners = cache.m_image_page_table.Find(static_cast( + address >> TextureCache::ImagePageTable::kPageBits)); + return owners != nullptr && owners->Erase(id); + } - static void SetQueryEpoch(TextureCache& cache, uint32_t epoch) { - std::lock_guard lock(cache.m_lock); - cache.m_image_query_epoch = epoch; - } + static void SetQueryEpoch(TextureCache &cache, uint32_t epoch) { + std::lock_guard lock(cache.m_lock); + cache.m_image_query_epoch = epoch; + } - static uint32_t QueryEpoch(TextureCache& cache) { - std::lock_guard lock(cache.m_lock); - return cache.m_image_query_epoch; - } + static uint32_t QueryEpoch(TextureCache &cache) { + std::lock_guard lock(cache.m_lock); + return cache.m_image_query_epoch; + } - static ImageId InsertImage(TextureCache& cache, const ImageInfo& info) { - std::lock_guard lock(cache.m_lock); - return cache.InsertImage(info); - } + static ImageId InsertImage(TextureCache &cache, const ImageInfo &info) { + std::lock_guard lock(cache.m_lock); + return cache.InsertImage(info); + } - static void DeleteImage(TextureCache& cache, ImageId id) { - std::lock_guard lock(cache.m_lock); - cache.DeleteImage(id); - } + static void DeleteImage(TextureCache &cache, ImageId id) { + std::lock_guard lock(cache.m_lock); + cache.DeleteImage(id); + } - static std::shared_ptr Owner(const TextureCache& cache, ImageId id) { - return cache.ResolveOwner(id); - } + static std::shared_ptr Owner(const TextureCache &cache, ImageId id) { + return cache.ResolveOwner(id); + } - static bool PendingDownload(const TextureCache& cache, ImageId id) { - return cache.m_download_images.contains(id); - } + static bool PendingDownload(const TextureCache &cache, ImageId id) { + return cache.m_download_images.contains(id); + } - static size_t NullImageCount(const TextureCache& cache) { return cache.m_null_images.size(); } + static size_t NullImageCount(const TextureCache &cache) { + return cache.m_null_images.size(); + } - static void TrackDownload(TextureCache& cache, ImageId id) { cache.TrackImageDownload(id); } + static void TrackDownload(TextureCache &cache, ImageId id) { + cache.TrackImageDownload(id); + } - static void AssociateStencil(TextureCache& cache, ImageId depth, GuestRange stencil) { - cache.AssociateStencil(depth, stencil); - } + static void AssociateStencil(TextureCache &cache, ImageId depth, + GuestRange stencil) { + cache.AssociateStencil(depth, stencil); + } - static void SetLinearReadback(TextureCache& cache, bool enabled) { - cache.m_readback_linear_images = enabled; - } + static void SetLinearReadback(TextureCache &cache, bool enabled) { + cache.m_readback_linear_images = enabled; + } - static std::pair MapDownload(TextureCache& cache, uint64_t size, - uint64_t alignment) { - return cache.MapDownload(size, alignment); - } + static std::pair + MapDownload(TextureCache &cache, uint64_t size, uint64_t alignment) { + return cache.MapDownload(size, alignment); + } - static bool TryDownload(TextureCache& cache, ImageId id) { return cache.TryDownloadImage(id); } + static bool TryDownload(TextureCache &cache, ImageId id) { + return cache.TryDownloadImage(id); + } - static TileManager& Tiler(TextureCache& cache) { return *cache.m_tiler; } + static TileManager &Tiler(TextureCache &cache) { return *cache.m_tiler; } }; struct RenderExecutorTestAccess { - static DescriptorCache::TextureBinding - ResolveTexture(RenderExecutor& executor, const ShaderRecompiler::IR::ImageResource& resource, - const ShaderRecompiler::IR::DescriptorValue& value) { - return executor.ResolveTexture(resource, value); - } + static DescriptorCache::TextureBinding + ResolveTexture(RenderExecutor &executor, + const ShaderRecompiler::IR::ImageResource &resource, + const ShaderRecompiler::IR::DescriptorValue &value) { + return executor.ResolveTexture(resource, value); + } - static auto PrepareGraphicsBindings(RenderExecutor& executor, CommandBuffer& buffer, - const ShaderStageRuntime& vertex, - const ShaderStageRuntime& pixel, bool pixel_active) { - return executor.PrepareGraphicsBindings(buffer, vertex, pixel, pixel_active); - } + static auto PrepareGraphicsBindings(RenderExecutor &executor, + CommandBuffer &buffer, + const ShaderStageRuntime &vertex, + const ShaderStageRuntime &pixel, + bool pixel_active) { + return executor.PrepareGraphicsBindings(buffer, vertex, pixel, + pixel_active); + } - static void CommitBindings(RenderExecutor& executor, CommandBuffer& buffer, - DescriptorCache::PreparedBindings& bindings) { - executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, nullptr, bindings); - } + static void CommitBindings(RenderExecutor &executor, CommandBuffer &buffer, + DescriptorCache::PreparedBindings &bindings) { + executor.CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, nullptr, + bindings); + } - static void ResolveRenderDepthTarget(RenderExecutor& executor, uint64_t submit_id, - RenderCommandBuffer& buffer, RenderDepthInfo& depth) { - executor.ResolveRenderDepthTarget(submit_id, buffer, depth); - } + static void ResolveRenderDepthTarget(RenderExecutor &executor, + uint64_t submit_id, + RenderCommandBuffer &buffer, + RenderDepthInfo &depth) { + executor.ResolveRenderDepthTarget(submit_id, buffer, depth); + } - static void ResolveRenderColorTarget(RenderExecutor& executor, uint64_t submit_id, - RenderCommandBuffer& buffer, RenderColorInfo& color, - uint32_t slot) { - executor.ResolveRenderColorTarget(submit_id, buffer, color, 0, slot); - } + static void ResolveRenderColorTarget(RenderExecutor &executor, + uint64_t submit_id, + RenderCommandBuffer &buffer, + RenderColorInfo &color, uint32_t slot) { + executor.ResolveRenderColorTarget(submit_id, buffer, color, 0, slot); + } - static void BindRenderTarget(RenderExecutor& executor, ImageId id) { - executor.BindRenderTarget(id); - } + static void BindRenderTarget(RenderExecutor &executor, ImageId id) { + executor.BindRenderTarget(id); + } - static RenderState AcquireRenderTargets(RenderExecutor& executor, CommandBuffer& buffer, - RenderColorInfo* colors, uint32_t color_count, - RenderDepthInfo& depth) { - return executor.AcquireRenderTargets(buffer, colors, color_count, depth); - } + static RenderState AcquireRenderTargets(RenderExecutor &executor, + CommandBuffer &buffer, + RenderColorInfo *colors, + uint32_t color_count, + RenderDepthInfo &depth) { + return executor.AcquireRenderTargets(buffer, colors, color_count, depth); + } - static void ResetBindings(RenderExecutor& executor) { executor.ResetBindings(); } + static void ResetBindings(RenderExecutor &executor) { + executor.ResetBindings(); + } - static bool BoundImagesInOrder(const RenderExecutor& executor, - const std::shared_ptr& first, - const std::shared_ptr& second) { - return executor.m_bound_images.size() == 2 && executor.m_bound_images[0] == first && - executor.m_bound_images[1] == second; - } + static bool BoundImagesInOrder(const RenderExecutor &executor, + const std::shared_ptr &first, + const std::shared_ptr &second) { + return executor.m_bound_images.size() == 2 && + executor.m_bound_images[0] == first && + executor.m_bound_images[1] == second; + } }; struct DescriptorCacheTestAccess { - static vk::DescriptorImageInfo MakeImageInfo(const DescriptorCache::TextureBinding& binding) { - return DescriptorCache::MakeImageInfo(binding); - } + static vk::DescriptorImageInfo + MakeImageInfo(const DescriptorCache::TextureBinding &binding) { + return DescriptorCache::MakeImageInfo(binding); + } }; namespace { -using u32 = uint32_t; -using BindingType = TextureCache::BindingType; -using ImageDesc = TextureCache::ImageDesc; +using u32 = uint32_t; +using BindingType = TextureCache::BindingType; +using ImageDesc = TextureCache::ImageDesc; using ShaderOpcode = ShaderRecompiler::Decoder::Opcode; -constexpr u32 InlineU32(u32 value) { - return 128u + value; -} +constexpr u32 InlineU32(u32 value) { return 128u + value; } -constexpr u32 Vgpr(u32 reg) { - return 256u + reg; -} +constexpr u32 Vgpr(u32 reg) { return 256u + reg; } constexpr u32 EncodeSMovB32(u32 dst, u32 src) { - return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) | (src & 0xffu); + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) | + (src & 0xffu); } constexpr u32 EncodeSop1(u32 opcode, u32 dst, u32 src) { - return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | ((opcode & 0xffu) << 8u) | - (src & 0xffu); + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | + ((opcode & 0xffu) << 8u) | (src & 0xffu); } constexpr u32 EncodeSop2(u32 opcode, u32 dst, u32 src0, u32 src1) { - return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | - ((src1 & 0xffu) << 8u) | (src0 & 0xffu); + return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); } constexpr u32 EncodeSopc(u32 opcode, u32 src0, u32 src1) { - return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) | ((src1 & 0xffu) << 8u) | - (src0 & 0xffu); + return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); } constexpr u32 EncodeSopp(u32 opcode, u32 simm = 0) { - return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) | (simm & 0xffffu); + return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) | + (simm & 0xffffu); } constexpr u32 EncodeSopk(u32 opcode, u32 dst, u32 imm) { - return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | - (imm & 0xffffu); + return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) | + ((dst & 0x7fu) << 16u) | (imm & 0xffffu); } constexpr u32 EncodeVop1(u32 opcode, u32 dst, u32 src0) { - return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) | (src0 & 0x1ffu); + return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) | + (src0 & 0x1ffu); } -constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0, u32 src0_sel = 6, - u32 src0_sext = 0, u32 src0_neg = 0, u32 src0_abs = 0, u32 s0 = 0, +constexpr u32 EncodeVop1Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0, + u32 src0_sel = 6, u32 src0_sext = 0, + u32 src0_neg = 0, u32 src0_abs = 0, u32 s0 = 0, u32 clamp = 0, u32 omod = 0) { - return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | - ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) | ((src0_sel & 0x7u) << 16u) | - ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | - ((s0 & 0x1u) << 23u); + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u); } constexpr u32 EncodeVop2(u32 opcode, u32 dst, u32 src0, u32 src1) { - return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | - (src0 & 0x1ffu); + return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | + ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu); } -constexpr u32 EncodeVop2Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0, u32 src0_sel = 6, - u32 src1_sel = 6, u32 src0_sext = 0, u32 src1_sext = 0, - u32 src0_neg = 0, u32 src0_abs = 0, u32 src1_neg = 0, u32 src1_abs = 0, - u32 s0 = 0, u32 s1 = 0, u32 clamp = 0, u32 omod = 0) { - return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | - ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) | ((src0_sel & 0x7u) << 16u) | - ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | - ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | ((src1_sext & 0x1u) << 27u) | - ((src1_neg & 0x1u) << 28u) | ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); +constexpr u32 EncodeVop2Sdwa(u32 src0, u32 dst_sel = 6, u32 dst_u = 0, + u32 src0_sel = 6, u32 src1_sel = 6, + u32 src0_sext = 0, u32 src1_sext = 0, + u32 src0_neg = 0, u32 src0_abs = 0, + u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0, + u32 s1 = 0, u32 clamp = 0, u32 omod = 0) { + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((clamp & 0x1u) << 13u) | ((omod & 0x3u) << 14u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | + ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); } -constexpr u32 EncodeVop2Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf, u32 bank_mask = 0xf, - u32 src0_neg = 0, u32 src0_abs = 0, u32 src1_neg = 0, +constexpr u32 EncodeVop2Dpp(u32 src0, u32 dpp_ctrl = 0, u32 row_mask = 0xf, + u32 bank_mask = 0xf, u32 src0_neg = 0, + u32 src0_abs = 0, u32 src1_neg = 0, u32 src1_abs = 0) { - return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | ((src0_neg & 0x1u) << 20u) | - ((src0_abs & 0x1u) << 21u) | ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) | - ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); + return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) | + ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); } constexpr u32 EncodeVop3Word0(u32 opcode, u32 dst, u32 abs = 0, u32 op_sel = 0, bool clamp = false) { - return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((abs & 0x7u) << 8u) | - ((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u) | (dst & 0xffu); + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((abs & 0x7u) << 8u) | + ((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u) | (dst & 0xffu); } constexpr u32 EncodeVop3BWord0(u32 opcode, u32 vdst, u32 sdst) { - return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) | (vdst & 0xffu); + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) | + (vdst & 0xffu); } -constexpr u32 EncodeVop3Word1(u32 src0, u32 src1, u32 src2 = 0, u32 omod = 0, u32 neg = 0) { - return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | - ((omod & 0x3u) << 27u) | ((neg & 0x7u) << 29u); +constexpr u32 EncodeVop3Word1(u32 src0, u32 src1, u32 src2 = 0, u32 omod = 0, + u32 neg = 0) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | + ((omod & 0x3u) << 27u) | ((neg & 0x7u) << 29u); } -constexpr u32 EncodeVop3pWord0(u32 opcode, u32 dst, u32 op_sel_hi = 0, u32 op_sel = 0, - u32 neg_hi = 0, bool clamp = false) { - return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | ((neg_hi & 0x7u) << 8u) | - ((op_sel & 0x7u) << 11u) | ((op_sel_hi & 0x4u) << 12u) | (clamp ? (1u << 15u) : 0u) | - (dst & 0xffu); +constexpr u32 EncodeVop3pWord0(u32 opcode, u32 dst, u32 op_sel_hi = 0, + u32 op_sel = 0, u32 neg_hi = 0, + bool clamp = false) { + return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | ((neg_hi & 0x7u) << 8u) | + ((op_sel & 0x7u) << 11u) | ((op_sel_hi & 0x4u) << 12u) | + (clamp ? (1u << 15u) : 0u) | (dst & 0xffu); } -constexpr u32 EncodeVop3pWord1(u32 src0, u32 src1, u32 src2 = 0, u32 op_sel_hi = 0, u32 neg = 0) { - return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | - ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u); +constexpr u32 EncodeVop3pWord1(u32 src0, u32 src1, u32 src2 = 0, + u32 op_sel_hi = 0, u32 neg = 0) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | + ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u); } constexpr u32 EncodeVopc(u32 opcode, u32 src0, u32 src1) { - return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu); + return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | + (src0 & 0x1ffu); } -constexpr u32 EncodeVopcSdwa(u32 src0, u32 sdst = 0, u32 sd = 0, u32 src0_sel = 6, u32 src1_sel = 6, - u32 src0_sext = 0, u32 src1_sext = 0, u32 src0_neg = 0, - u32 src0_abs = 0, u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0, +constexpr u32 EncodeVopcSdwa(u32 src0, u32 sdst = 0, u32 sd = 0, + u32 src0_sel = 6, u32 src1_sel = 6, + u32 src0_sext = 0, u32 src1_sext = 0, + u32 src0_neg = 0, u32 src0_abs = 0, + u32 src1_neg = 0, u32 src1_abs = 0, u32 s0 = 0, u32 s1 = 0) { - return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) | - ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | - ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | - ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | ((src1_abs & 0x1u) << 29u) | - ((s1 & 0x1u) << 31u); + return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | + ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); } -constexpr u32 EncodeMubuf0(u32 opcode, u32 offset = 0, bool idxen = false, bool offen = true, - bool glc = false) { - return (0x38u << 26u) | ((opcode & 0x7fu) << 18u) | (offen ? (1u << 12u) : 0u) | - (idxen ? (1u << 13u) : 0u) | (glc ? (1u << 14u) : 0u) | (offset & 0xfffu); +constexpr u32 EncodeMubuf0(u32 opcode, u32 offset = 0, bool idxen = false, + bool offen = true, bool glc = false) { + return (0x38u << 26u) | ((opcode & 0x7fu) << 18u) | + (offen ? (1u << 12u) : 0u) | (idxen ? (1u << 13u) : 0u) | + (glc ? (1u << 14u) : 0u) | (offset & 0xfffu); } constexpr u32 EncodeMubuf1(u32 vdata, u32 srsrc, u32 vaddr, u32 soffset = 128) { - return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | - (vaddr & 0xffu); + return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); } -constexpr u32 EncodeMtbuf0(u32 opcode, u32 dfmt, u32 nfmt, u32 offset = 0, bool idxen = false, - bool offen = true) { - return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) | - (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) | ((dfmt & 0xfu) << 19u) | - ((nfmt & 0x7u) << 23u); +constexpr u32 EncodeMtbuf0(u32 opcode, u32 dfmt, u32 nfmt, u32 offset = 0, + bool idxen = false, bool offen = true) { + return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) | + (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) | + ((dfmt & 0xfu) << 19u) | ((nfmt & 0x7u) << 23u); } -constexpr u32 EncodeMtbuf1(u32 opcode, u32 vdata, u32 srsrc, u32 vaddr, u32 soffset = 128) { - return (((opcode >> 3u) & 1u) << 21u) | ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | - ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); +constexpr u32 EncodeMtbuf1(u32 opcode, u32 vdata, u32 srsrc, u32 vaddr, + u32 soffset = 128) { + return (((opcode >> 3u) & 1u) << 21u) | ((soffset & 0xffu) << 24u) | + ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); } constexpr u32 EncodeSmem0(u32 opcode, u32 dst, u32 sbase = 0) { - return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) | (sbase & 0x3fu); + return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) | + (sbase & 0x3fu); } constexpr u32 EncodeSmem1(u32 offset, u32 soffset = 0) { - return (offset & 0x1fffffu) | ((soffset & 0x7fu) << 25u); + return (offset & 0x1fffffu) | ((soffset & 0x7fu) << 25u); } -constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0, bool glc = false, - u32 dim = 1) { - return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) | ((nsa_dwords & 0x3u) << 1u) | - ((dim & 0x7u) << 3u) | ((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) | - ((opcode & 0x7fu) << 18u); +constexpr u32 EncodeMimg0(u32 opcode, u32 dmask, u32 nsa_dwords = 0, + bool glc = false, u32 dim = 1) { + return (0x3cu << 26u) | ((opcode >> 7u) & 0x1u) | + ((nsa_dwords & 0x3u) << 1u) | ((dim & 0x7u) << 3u) | + ((dmask & 0xfu) << 8u) | (glc ? (1u << 13u) : 0u) | + ((opcode & 0x7fu) << 18u); } -constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0, bool a16 = false) { - return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | - (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u); +constexpr u32 EncodeMimg1(u32 vdata, u32 vaddr, u32 srsrc = 0, u32 ssamp = 0, + bool a16 = false) { + return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u); } constexpr u32 EncodeVintrp(u32 opcode, u32 dst, u32 attr, u32 chan, u32 src) { - return (0x32u << 26u) | ((dst & 0xffu) << 18u) | ((opcode & 0x3u) << 16u) | - ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (src & 0xffu); + return (0x32u << 26u) | ((dst & 0xffu) << 18u) | ((opcode & 0x3u) << 16u) | + ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (src & 0xffu); } -constexpr u32 EncodeExp0(u32 target, u32 en, bool done = true, bool compr = false, - bool vm = false) { - return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) | (compr ? (1u << 10u) : 0u) | - (done ? (1u << 11u) : 0u) | (vm ? (1u << 12u) : 0u); +constexpr u32 EncodeExp0(u32 target, u32 en, bool done = true, + bool compr = false, bool vm = false) { + return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) | + (compr ? (1u << 10u) : 0u) | (done ? (1u << 11u) : 0u) | + (vm ? (1u << 12u) : 0u); } constexpr u32 EncodeExp1(u32 src0, u32 src1, u32 src2, u32 src3) { - return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) | - ((src3 & 0xffu) << 24u); + return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) | + ((src3 & 0xffu) << 24u); } constexpr u32 EncodeFlat0(u32 opcode, u32 segment, u32 offset = 0) { - return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) | ((segment & 0x3u) << 14u) | - (offset & 0xfffu); + return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) | + ((segment & 0x3u) << 14u) | (offset & 0xfffu); } constexpr u32 EncodeFlat1(u32 vdst, u32 saddr, u32 data, u32 addr) { - return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) | ((data & 0xffu) << 8u) | - (addr & 0xffu); + return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) | + ((data & 0xffu) << 8u) | (addr & 0xffu); } constexpr u32 EncodeDs0(u32 opcode, u32 offset = 0, bool gds = false) { - return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (gds ? (1u << 17u) : 0u) | - (offset & 0xffffu); + return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (gds ? (1u << 17u) : 0u) | + (offset & 0xffffu); } constexpr u32 EncodeDs1Ex(u32 vdst, u32 data1, u32 data0, u32 addr) { - return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) | ((data0 & 0xffu) << 8u) | - (addr & 0xffu); + return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) | + ((data0 & 0xffu) << 8u) | (addr & 0xffu); } constexpr u32 EncodeDs1(u32 vdst, u32 data0, u32 addr) { - return EncodeDs1Ex(vdst, 0, data0, addr); + return EncodeDs1Ex(vdst, 0, data0, addr); } -void AppendSMovLiteral(std::vector* code, u32 dst, u32 literal) { - code->push_back(EncodeSMovB32(dst, 255u)); - code->push_back(literal); +void AppendSMovLiteral(std::vector *code, u32 dst, u32 literal) { + code->push_back(EncodeSMovB32(dst, 255u)); + code->push_back(literal); } -void AppendVMovLiteral(std::vector* code, u32 dst, u32 literal) { - code->push_back(EncodeVop1(0x01, dst, 255u)); - code->push_back(literal); +void AppendVMovLiteral(std::vector *code, u32 dst, u32 literal) { + code->push_back(EncodeVop1(0x01, dst, 255u)); + code->push_back(literal); } -void AppendVMovU32(std::vector* code, u32 dst, u32 value) { - if (value <= 64u) { - code->push_back(EncodeVop1(0x01, dst, InlineU32(value))); - return; - } - AppendVMovLiteral(code, dst, value); +void AppendVMovU32(std::vector *code, u32 dst, u32 value) { + if (value <= 64u) { + code->push_back(EncodeVop1(0x01, dst, InlineU32(value))); + return; + } + AppendVMovLiteral(code, dst, value); } -void AppendVop3(std::vector* code, u32 opcode, u32 dst, u32 src0, u32 src1, u32 src2 = 0, - u32 abs = 0, u32 op_sel = 0, bool clamp = false, u32 omod = 0, u32 neg = 0) { - code->push_back(EncodeVop3Word0(opcode, dst, abs, op_sel, clamp)); - code->push_back(EncodeVop3Word1(src0, src1, src2, omod, neg)); +void AppendVop3(std::vector *code, u32 opcode, u32 dst, u32 src0, u32 src1, + u32 src2 = 0, u32 abs = 0, u32 op_sel = 0, bool clamp = false, + u32 omod = 0, u32 neg = 0) { + code->push_back(EncodeVop3Word0(opcode, dst, abs, op_sel, clamp)); + code->push_back(EncodeVop3Word1(src0, src1, src2, omod, neg)); } -void AppendVop3B(std::vector* code, u32 opcode, u32 vdst, u32 sdst, u32 src0, u32 src1, - u32 src2 = 0) { - code->push_back(EncodeVop3BWord0(opcode, vdst, sdst)); - code->push_back(EncodeVop3Word1(src0, src1, src2)); +void AppendVop3B(std::vector *code, u32 opcode, u32 vdst, u32 sdst, + u32 src0, u32 src1, u32 src2 = 0) { + code->push_back(EncodeVop3BWord0(opcode, vdst, sdst)); + code->push_back(EncodeVop3Word1(src0, src1, src2)); } -void AppendVop3p(std::vector* code, u32 opcode, u32 dst, u32 src0, u32 src1, u32 src2 = 0, - u32 op_sel_hi = 0, u32 op_sel = 0, u32 neg_hi = 0, u32 neg = 0) { - code->push_back(EncodeVop3pWord0(opcode, dst, op_sel_hi, op_sel, neg_hi)); - code->push_back(EncodeVop3pWord1(src0, src1, src2, op_sel_hi, neg)); +void AppendVop3p(std::vector *code, u32 opcode, u32 dst, u32 src0, + u32 src1, u32 src2 = 0, u32 op_sel_hi = 0, u32 op_sel = 0, + u32 neg_hi = 0, u32 neg = 0) { + code->push_back(EncodeVop3pWord0(opcode, dst, op_sel_hi, op_sel, neg_hi)); + code->push_back(EncodeVop3pWord1(src0, src1, src2, op_sel_hi, neg)); } -void AppendBufferLoadDword(std::vector* code, u32 dst_vgpr, u32 address_vgpr) { - code->push_back(EncodeMubuf0(0x0cu)); - code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr)); +void AppendBufferLoadDword(std::vector *code, u32 dst_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMubuf0(0x0cu)); + code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr)); } -void AppendBufferLoadOpcode(std::vector* code, u32 opcode, u32 dst_vgpr, u32 address_vgpr) { - code->push_back(EncodeMubuf0(opcode)); - code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr)); +void AppendBufferLoadOpcode(std::vector *code, u32 opcode, u32 dst_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMubuf0(opcode)); + code->push_back(EncodeMubuf1(dst_vgpr, 0, address_vgpr)); } -void AppendBufferStoreDword(std::vector* code, u32 value_vgpr, u32 address_vgpr) { - code->push_back(EncodeMubuf0(0x1cu)); - code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr)); +void AppendBufferStoreDword(std::vector *code, u32 value_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMubuf0(0x1cu)); + code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr)); } -void AppendBufferStoreOpcode(std::vector* code, u32 opcode, u32 value_vgpr, u32 address_vgpr, - bool glc = false) { - code->push_back(EncodeMubuf0(opcode, 0, false, true, glc)); - code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr)); +void AppendBufferStoreOpcode(std::vector *code, u32 opcode, u32 value_vgpr, + u32 address_vgpr, bool glc = false) { + code->push_back(EncodeMubuf0(opcode, 0, false, true, glc)); + code->push_back(EncodeMubuf1(value_vgpr, 12, address_vgpr)); } -void AppendTBufferLoadOpcode(std::vector* code, u32 opcode, u32 dst_vgpr, u32 address_vgpr) { - code->push_back(EncodeMtbuf0(opcode, 14, 7)); - code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr)); +void AppendTBufferLoadOpcode(std::vector *code, u32 opcode, u32 dst_vgpr, + u32 address_vgpr) { + code->push_back(EncodeMtbuf0(opcode, 14, 7)); + code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr)); } constexpr u32 BufferFormat(Prospero::BufferFormat format) { - return static_cast(format); + return static_cast(format); } -void AppendTBufferLoadFormatOpcode(std::vector* code, u32 opcode, u32 dst_vgpr, - u32 address_vgpr, Prospero::BufferFormat format) { - const auto value = BufferFormat(format); - code->push_back(EncodeMtbuf0(opcode, value & 0xfu, (value >> 4u) & 0x7u)); - code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr)); +void AppendTBufferLoadFormatOpcode(std::vector *code, u32 opcode, + u32 dst_vgpr, u32 address_vgpr, + Prospero::BufferFormat format) { + const auto value = BufferFormat(format); + code->push_back(EncodeMtbuf0(opcode, value & 0xfu, (value >> 4u) & 0x7u)); + code->push_back(EncodeMtbuf1(opcode, dst_vgpr, 0, address_vgpr)); } -void AppendTBufferStoreOpcode(std::vector* code, u32 opcode, u32 value_vgpr, - u32 address_vgpr) { - code->push_back(EncodeMtbuf0(opcode, 14, 7)); - code->push_back(EncodeMtbuf1(opcode, value_vgpr, 0, address_vgpr)); +void AppendTBufferStoreOpcode(std::vector *code, u32 opcode, + u32 value_vgpr, u32 address_vgpr) { + code->push_back(EncodeMtbuf0(opcode, 14, 7)); + code->push_back(EncodeMtbuf1(opcode, value_vgpr, 0, address_vgpr)); } -void AppendSmemLoadOpcode(std::vector* code, u32 opcode, u32 dst_sgpr, u32 byte_offset) { - code->push_back(EncodeSmem0(opcode, dst_sgpr)); - code->push_back(EncodeSmem1(byte_offset)); +void AppendSmemLoadOpcode(std::vector *code, u32 opcode, u32 dst_sgpr, + u32 byte_offset) { + code->push_back(EncodeSmem0(opcode, dst_sgpr)); + code->push_back(EncodeSmem1(byte_offset)); } -void AppendStoreVgpr(std::vector* code, u32 value_vgpr, u32 dword_index) { - AppendVMovU32(code, 31, dword_index * 4u); - AppendBufferStoreDword(code, value_vgpr, 31); +void AppendStoreVgpr(std::vector *code, u32 value_vgpr, u32 dword_index) { + AppendVMovU32(code, 31, dword_index * 4u); + AppendBufferStoreDword(code, value_vgpr, 31); } -void AppendStoreVgprAtLaneDwordOffset(std::vector* code, u32 value_vgpr, u32 lane_vgpr, - u32 dword_offset) { - if (dword_offset == 0u) { - code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), lane_vgpr)); - } else { - AppendVMovU32(code, 31, dword_offset); - code->push_back(EncodeVop2(0x25, 31, Vgpr(lane_vgpr), 31)); - code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), 31)); - } - AppendBufferStoreDword(code, value_vgpr, 31); +void AppendStoreVgprAtLaneDwordOffset(std::vector *code, u32 value_vgpr, + u32 lane_vgpr, u32 dword_offset) { + if (dword_offset == 0u) { + code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), lane_vgpr)); + } else { + AppendVMovU32(code, 31, dword_offset); + code->push_back(EncodeVop2(0x25, 31, Vgpr(lane_vgpr), 31)); + code->push_back(EncodeVop2(0x1a, 31, InlineU32(2), 31)); + } + AppendBufferStoreDword(code, value_vgpr, 31); } -void AppendStoreSgpr(std::vector* code, u32 value_sgpr, u32 dword_index) { - code->push_back(EncodeVop1(0x01, 30, value_sgpr)); - AppendStoreVgpr(code, 30, dword_index); +void AppendStoreSgpr(std::vector *code, u32 value_sgpr, u32 dword_index) { + code->push_back(EncodeVop1(0x01, 30, value_sgpr)); + AppendStoreVgpr(code, 30, dword_index); } -void AppendStoreSgprAtLaneDwordOffset(std::vector* code, u32 value_sgpr, u32 lane_vgpr, - u32 dword_offset) { - code->push_back(EncodeVop1(0x01, 30, value_sgpr)); - AppendStoreVgprAtLaneDwordOffset(code, 30, lane_vgpr, dword_offset); +void AppendStoreSgprAtLaneDwordOffset(std::vector *code, u32 value_sgpr, + u32 lane_vgpr, u32 dword_offset) { + code->push_back(EncodeVop1(0x01, 30, value_sgpr)); + AppendStoreVgprAtLaneDwordOffset(code, 30, lane_vgpr, dword_offset); } -void AppendStoreSgprPair(std::vector* code, u32 value_sgpr, u32 first_dword_index) { - AppendStoreSgpr(code, value_sgpr, first_dword_index); - AppendStoreSgpr(code, value_sgpr + 1u, first_dword_index + 1u); +void AppendStoreSgprPair(std::vector *code, u32 value_sgpr, + u32 first_dword_index) { + AppendStoreSgpr(code, value_sgpr, first_dword_index); + AppendStoreSgpr(code, value_sgpr + 1u, first_dword_index + 1u); } -void AppendEnd(std::vector* code) { - code->push_back(0xbf810000u); -} +void AppendEnd(std::vector *code) { code->push_back(0xbf810000u); } std::string Hex(u32 value) { - char buffer[32] = {}; - std::snprintf(buffer, sizeof(buffer), "0x%08" PRIx32, value); - return buffer; + char buffer[32] = {}; + std::snprintf(buffer, sizeof(buffer), "0x%08" PRIx32, value); + return buffer; } std::string VulkanResultName(vk::Result result) { - return VulkanToString(result); + return VulkanToString(result); } -[[noreturn]] void Fail(const char* shader_name, const char* stage, const std::string& message) { - std::fprintf(stderr, "ShaderRecompilerComputeTests: %s failed at %s: %s\n", shader_name, stage, - message.c_str()); - std::abort(); +[[noreturn]] void Fail(const char *shader_name, const char *stage, + const std::string &message) { + std::fprintf(stderr, "ShaderRecompilerComputeTests: %s failed at %s: %s\n", + shader_name, stage, message.c_str()); + std::abort(); } -void Require(const char* shader_name, const char* stage, bool value, const std::string& message); +void Require(const char *shader_name, const char *stage, bool value, + const std::string &message); void EnsureConfigInitialized() { - static bool config_initialized = false; - if (!config_initialized) { - static Common::Subsystems subsystems; - Common::InitializeThreads(); - subsystems.Initialize(); - Config::ConfigOptions options; - options.printf_direction = Config::OutputDirection::Silent; - Config::Load(options); - subsystems.Initialize(); - subsystems.Initialize(); - config_initialized = true; - } + static bool config_initialized = false; + if (!config_initialized) { + static Common::Subsystems subsystems; + Common::InitializeThreads(); + subsystems.Initialize(); + Config::ConfigOptions options; + options.printf_direction = Config::OutputDirection::Silent; + Config::Load(options); + subsystems.Initialize(); + subsystems.Initialize(); + config_initialized = true; + } } -void RequireVk(const char* shader_name, const char* stage, vk::Result result, const char* action) { - if (result != vk::Result::eSuccess) { - Fail(shader_name, stage, std::string(action) + " returned " + VulkanResultName(result)); - } +void RequireVk(const char *shader_name, const char *stage, vk::Result result, + const char *action) { + if (result != vk::Result::eSuccess) { + Fail(shader_name, stage, + std::string(action) + " returned " + VulkanResultName(result)); + } } -void Require(const char* shader_name, const char* stage, bool value, const std::string& message) { - if (!value) { - Fail(shader_name, stage, message); - } +void Require(const char *shader_name, const char *stage, bool value, + const std::string &message) { + if (!value) { + Fail(shader_name, stage, message); + } } void CheckLeastRecentlyUsedCacheOrdering() { - Common::LeastRecentlyUsedCache cache; - const auto first = cache.Insert(1, 1); - const auto middle = cache.Insert(2, 2); - const auto last = cache.Insert(3, 3); - (void)first; - (void)last; - cache.Touch(middle, 4); - std::vector visited; - cache.ForEachItemBelow(4, [&](uint32_t value) { - visited.push_back(value); - return visited.size() > 3; - }); - Require("LeastRecentlyUsedCache", "non-tail touch traversal", - visited == std::vector {1, 3, 2}, - "touching a non-tail item left a cycle or changed LRU order"); - std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache"); + Common::LeastRecentlyUsedCache cache; + const auto first = cache.Insert(1, 1); + const auto middle = cache.Insert(2, 2); + const auto last = cache.Insert(3, 3); + (void)first; + (void)last; + cache.Touch(middle, 4); + std::vector visited; + cache.ForEachItemBelow(4, [&](uint32_t value) { + visited.push_back(value); + return visited.size() > 3; + }); + Require("LeastRecentlyUsedCache", "non-tail touch traversal", + visited == std::vector{1, 3, 2}, + "touching a non-tail item left a cycle or changed LRU order"); + std::printf("[host] %-32s ok\n", "LeastRecentlyUsedCache"); } struct TestCase { - const char* name = ""; - std::vector code; - std::vector initial; - std::vector expected; - std::vector opcodes; - u32 image_width = 4; - u32 image_height = 4; - std::vector sampled_image_rgba; - std::vector> sampled_image_rgba_mips; - vk::Format sampled_image_format = vk::Format::eR32G32B32A32Sfloat; - u32 sampled_image_dwords_per_pixel = 4; - vk::ImageType sampled_image_type = vk::ImageType::e2D; - vk::ImageViewType sampled_image_view_type = vk::ImageViewType::e2D; - u32 sampled_image_layers = 1; - u32 sampled_image_view_base_layer = 0; - u32 sampled_image_view_layers = 0; - std::vector storage_image_rgba; - std::vector expected_storage_image_rgba; - vk::Format storage_image_format = vk::Format::eR32G32B32A32Sfloat; - u32 storage_image_dwords_per_pixel = 4; - std::vector storage_image_r32ui; - std::vector expected_storage_image_r32ui; - std::vector required_spirv; - std::vector forbidden_spirv; - ShaderComputeInputInfo compute_info {}; - bool has_compute_info = false; - u32 dispatch_x = 1; - u32 dispatch_y = 1; - u32 dispatch_z = 1; - std::array user_data {}; - bool has_user_data = false; - u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7); - bool compile_only = false; - size_t storage_buffer_range_dwords = 0; - std::vector storage_buffer_offsets; - bool force_shader_data_storage = false; - std::optional flat_memory_base; - std::vector gds_initial; - std::vector expected_gds; + const char *name = ""; + std::vector code; + std::vector initial; + std::vector expected; + std::vector opcodes; + u32 image_width = 4; + u32 image_height = 4; + std::vector sampled_image_rgba; + std::vector> sampled_image_rgba_mips; + vk::Format sampled_image_format = vk::Format::eR32G32B32A32Sfloat; + u32 sampled_image_dwords_per_pixel = 4; + vk::ImageType sampled_image_type = vk::ImageType::e2D; + vk::ImageViewType sampled_image_view_type = vk::ImageViewType::e2D; + u32 sampled_image_layers = 1; + u32 sampled_image_view_base_layer = 0; + u32 sampled_image_view_layers = 0; + std::vector storage_image_rgba; + std::vector expected_storage_image_rgba; + vk::Format storage_image_format = vk::Format::eR32G32B32A32Sfloat; + u32 storage_image_dwords_per_pixel = 4; + std::vector storage_image_r32ui; + std::vector expected_storage_image_r32ui; + std::vector required_spirv; + std::vector forbidden_spirv; + ShaderComputeInputInfo compute_info{}; + bool has_compute_info = false; + u32 dispatch_x = 1; + u32 dispatch_y = 1; + u32 dispatch_z = 1; + std::array user_data{}; + bool has_user_data = false; + u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7); + bool compile_only = false; + size_t storage_buffer_range_dwords = 0; + std::vector storage_buffer_offsets; + bool force_shader_data_storage = false; + std::optional flat_memory_base; + std::vector gds_initial; + std::vector expected_gds; + std::vector> decoded_counts; + std::vector> ir_counts; }; struct SkippedCase { - const char* name = ""; - const char* reason = ""; + const char *name = ""; + const char *reason = ""; }; struct GraphicsCase { - const char* name = ""; - std::vector fragment_code; - std::vector expected_pixel; - std::vector opcodes; - std::array user_data {}; - bool has_user_data = false; - std::vector push_constants; - std::vector pixel_interpolator_settings; - bool pixel_no_perspective = false; - std::vector vertices; + const char *name = ""; + std::vector fragment_code; + std::vector expected_pixel; + std::vector opcodes; + std::array user_data{}; + bool has_user_data = false; + std::vector push_constants; + std::vector pixel_interpolator_settings; + bool pixel_no_perspective = false; + std::vector vertices; }; struct CompiledShader { - std::vector spirv; - ShaderRecompiler::IR::Program program; - std::vector flattened_srt; - std::vector packed_user_data; + std::vector spirv; + ShaderRecompiler::IR::Program program; + std::vector flattened_srt; + std::vector packed_user_data; }; -std::array MakeNativeUserData(const std::array* source) { - std::array data {}; - data[50] = 1u << 20u; - if (source != nullptr) { - data = *source; - } - return data; +std::array MakeNativeUserData(const std::array *source) { + std::array data{}; + data[50] = 1u << 20u; + if (source != nullptr) { + data = *source; + } + return data; } -bool ReadTestMemory(void* userdata, uint64_t address, u32* value) { - const auto* data = static_cast*>(userdata); - if (data == nullptr || value == nullptr || address % 4u != 0 || address / 4u >= data->size()) { - return false; - } - *value = (*data)[address / 4u]; - return true; +bool ReadTestMemory(void *userdata, uint64_t address, u32 *value) { + const auto *data = static_cast *>(userdata); + if (data == nullptr || value == nullptr || address % 4u != 0 || + address / 4u >= data->size()) { + return false; + } + *value = (*data)[address / 4u]; + return true; } -void ValidateSpirv(const char* shader_name, const std::vector& spirv) { - spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); - std::string messages; - tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, - const spv_position_t& position, const char* message) { - char buffer[1024] = {}; - std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, position.column, - message); - messages += buffer; - }); +void ValidateSpirv(const char *shader_name, const std::vector &spirv) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string messages; + tools.SetMessageConsumer([&messages](spv_message_level_t, const char *, + const spv_position_t &position, + const char *message) { + char buffer[1024] = {}; + std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, + position.column, message); + messages += buffer; + }); - if (!tools.Validate(spirv)) { - Fail(shader_name, "SPIR-V validation", messages); - } + if (!tools.Validate(spirv)) { + Fail(shader_name, "SPIR-V validation", messages); + } } -size_t CountText(const std::string& text, const std::string& needle) { - size_t count = 0; - for (size_t offset = 0; (offset = text.find(needle, offset)) != std::string::npos; - offset += needle.size()) { - count++; - } - return count; +size_t CountText(const std::string &text, const std::string &needle) { + size_t count = 0; + for (size_t offset = 0; + (offset = text.find(needle, offset)) != std::string::npos; + offset += needle.size()) { + count++; + } + return count; } void CheckRectListShaders() { - constexpr const char* name = "RectListShaders"; + constexpr const char *name = "RectListShaders"; - auto program = std::make_shared(); - program->info.inputs.push_back( - {ShaderRecompiler::IR::StageInputKind::Parameter, 0, 4, "in_param_0"}); - program->info.inputs.push_back( - {ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"}); + auto program = std::make_shared(); + program->info.inputs.push_back( + {ShaderRecompiler::IR::StageInputKind::Parameter, 0, 4, "in_param_0"}); + program->info.inputs.push_back( + {ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"}); - ShaderVertexInputInfo vertex {}; - vertex.param_export_mask = 1u; - ShaderPixelInputInfo pixel {}; - pixel.input_num = 2; - pixel.interpolator_settings[0] = 0x400u; - pixel.interpolator_settings[1] = 0; - pixel.stage.program = program; - HW::PixelShaderInfo ps_regs {}; - const auto perspective_id = ShaderGetIdPS(ps_regs, pixel, false); - pixel.ps_no_perspective = true; - const auto no_perspective_id = ShaderGetIdPS(ps_regs, pixel, false); - pixel.ps_no_perspective = false; - Require(name, "pipeline identity", perspective_id != no_perspective_id, - "pixel interpolation mode must participate in the shader and pipeline key"); + ShaderVertexInputInfo vertex{}; + vertex.param_export_mask = 1u; + ShaderPixelInputInfo pixel{}; + pixel.input_num = 2; + pixel.interpolator_settings[0] = 0x400u; + pixel.interpolator_settings[1] = 0; + pixel.stage.program = program; + HW::PixelShaderInfo ps_regs{}; + const auto perspective_id = ShaderGetIdPS(ps_regs, pixel, false); + pixel.ps_no_perspective = true; + const auto no_perspective_id = ShaderGetIdPS(ps_regs, pixel, false); + pixel.ps_no_perspective = false; + Require(name, "pipeline identity", perspective_id != no_perspective_id, + "pixel interpolation mode must participate in the shader and " + "pipeline key"); - const std::array active_inputs = {0, 1}; - Require(name, "duplicate mapping", - ShaderPixelParameterLocation(pixel, active_inputs, 0) == 0 && - ShaderPixelParameterLocation(pixel, active_inputs, 1) == 1, - "duplicate pixel mappings must receive distinct effective output locations"); + const std::array active_inputs = {0, 1}; + Require(name, "duplicate mapping", + ShaderPixelParameterLocation(pixel, active_inputs, 0) == 0 && + ShaderPixelParameterLocation(pixel, active_inputs, 1) == 1, + "duplicate pixel mappings must receive distinct effective output " + "locations"); - const auto shaders = BuildRectListShaders(vertex, &pixel); - Require(name, "SPIR-V version", - shaders.control.size() > 1 && shaders.control[1] == 0x00010500u && - shaders.evaluation.size() > 1 && shaders.evaluation[1] == 0x00010500u, + const auto shaders = BuildRectListShaders(vertex, &pixel); + Require(name, "SPIR-V version", + shaders.control.size() > 1 && shaders.control[1] == 0x00010500u && + shaders.evaluation.size() > 1 && + shaders.evaluation[1] == 0x00010500u, "shadPS4-compatible vector selection requires SPIR-V 1.5"); - ValidateSpirv(name, shaders.control); - ValidateSpirv(name, shaders.evaluation); + ValidateSpirv(name, shaders.control); + ValidateSpirv(name, shaders.evaluation); - spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); - std::string control_text; - std::string evaluation_text; - Require(name, "control disassembly", tools.Disassemble(shaders.control, &control_text), - "failed to disassemble rectangle-list tessellation control shader"); - Require(name, "evaluation disassembly", - tools.Disassemble(shaders.evaluation, &evaluation_text), - "failed to disassemble rectangle-list tessellation evaluation shader"); - Require(name, "control execution mode", - control_text.find("TessellationControl") != std::string::npos && - control_text.find("OutputVertices 4") != std::string::npos, - "rectangle-list control shader must produce four control points"); - Require(name, "evaluation execution modes", - evaluation_text.find("TessellationEvaluation") != std::string::npos && - evaluation_text.find("Quads") != std::string::npos && - evaluation_text.find("SpacingEqual") != std::string::npos && - evaluation_text.find("VertexOrderCw") != std::string::npos, - "rectangle-list evaluation shader has the wrong patch modes"); - Require(name, "no geometry stage", - control_text.find("Geometry") == std::string::npos && - evaluation_text.find("Geometry") == std::string::npos, - "rectangle-list expansion must not use geometry shaders"); - Require(name, "flat broadcast", - CountText(control_text, "OpVectorTimesScalar") == 6 && - CountText(control_text, "OpSelect %v4float") == 2, - "flat parameters must use guest vertex zero instead of reconstructed values"); - Require(name, "remapped interface", - CountText(control_text, " Location 0") == 2 && - CountText(control_text, " Location 1") == 1 && - CountText(evaluation_text, " Location 0") == 2 && - CountText(evaluation_text, " Location 1") == 2, - "duplicate pixel mappings must share one vertex input and keep distinct patch outputs"); + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string control_text; + std::string evaluation_text; + Require(name, "control disassembly", + tools.Disassemble(shaders.control, &control_text), + "failed to disassemble rectangle-list tessellation control shader"); + Require( + name, "evaluation disassembly", + tools.Disassemble(shaders.evaluation, &evaluation_text), + "failed to disassemble rectangle-list tessellation evaluation shader"); + Require(name, "control execution mode", + control_text.find("TessellationControl") != std::string::npos && + control_text.find("OutputVertices 4") != std::string::npos, + "rectangle-list control shader must produce four control points"); + Require(name, "evaluation execution modes", + evaluation_text.find("TessellationEvaluation") != std::string::npos && + evaluation_text.find("Quads") != std::string::npos && + evaluation_text.find("SpacingEqual") != std::string::npos && + evaluation_text.find("VertexOrderCw") != std::string::npos, + "rectangle-list evaluation shader has the wrong patch modes"); + Require(name, "no geometry stage", + control_text.find("Geometry") == std::string::npos && + evaluation_text.find("Geometry") == std::string::npos, + "rectangle-list expansion must not use geometry shaders"); + Require(name, "flat broadcast", + CountText(control_text, "OpVectorTimesScalar") == 6 && + CountText(control_text, "OpSelect %v4float") == 2, + "flat parameters must use guest vertex zero instead of reconstructed " + "values"); + Require(name, "remapped interface", + CountText(control_text, " Location 0") == 2 && + CountText(control_text, " Location 1") == 1 && + CountText(evaluation_text, " Location 0") == 2 && + CountText(evaluation_text, " Location 1") == 2, + "duplicate pixel mappings must share one vertex input and keep " + "distinct patch outputs"); - const auto position_only = BuildRectListShaders(vertex, nullptr); - ValidateSpirv(name, position_only.control); - ValidateSpirv(name, position_only.evaluation); + const auto position_only = BuildRectListShaders(vertex, nullptr); + ValidateSpirv(name, position_only.control); + ValidateSpirv(name, position_only.evaluation); } -void CheckSpirvText(const TestCase& test, const std::vector& spirv) { - if (test.required_spirv.empty() && test.forbidden_spirv.empty()) { - return; - } +void CheckSpirvText(const TestCase &test, const std::vector &spirv) { + if (test.required_spirv.empty() && test.forbidden_spirv.empty()) { + return; + } - spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); - std::string text; - if (!tools.Disassemble(spirv, &text)) { - Fail(test.name, "SPIR-V disassembly", "failed to disassemble emitted SPIR-V"); - } - for (const auto& required: test.required_spirv) { - if (text.find(required) == std::string::npos) { - Fail(test.name, "SPIR-V disassembly", - std::string("missing required text: ") + required); - } - } - for (const auto& forbidden: test.forbidden_spirv) { - if (text.find(forbidden) != std::string::npos) { - Fail(test.name, "SPIR-V disassembly", - std::string("found forbidden text: ") + forbidden); - } - } + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string text; + if (!tools.Disassemble(spirv, &text)) { + Fail(test.name, "SPIR-V disassembly", + "failed to disassemble emitted SPIR-V"); + } + for (const auto &required : test.required_spirv) { + if (text.find(required) == std::string::npos) { + Fail(test.name, "SPIR-V disassembly", + std::string("missing required text: ") + required); + } + } + for (const auto &forbidden : test.forbidden_spirv) { + if (text.find(forbidden) != std::string::npos) { + Fail(test.name, "SPIR-V disassembly", + std::string("found forbidden text: ") + forbidden); + } + } } -CompiledShader CompileCase(const TestCase& test) { - auto user_data = MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); - const auto uses_image = std::any_of(test.opcodes.begin(), test.opcodes.end(), [](auto op) { - return op >= ShaderOpcode::ImageGetResinfo && op <= ShaderOpcode::ImageGather4H; - }); - if (uses_image && ((user_data[3] >> 28u) & 0xfu) == 0) { - user_data[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; - } - if (uses_image) { - user_data[3] = (user_data[3] & ~0xfffu) | test.image_descriptor_swizzle; - } - if (!test.has_user_data) { - user_data[2] = static_cast(test.initial.size() * sizeof(u32)); - } - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.compute_input_info = test.has_compute_info ? &test.compute_info : nullptr; - options.user_data = user_data.data(); - options.read_memory = ReadTestMemory; - options.read_memory_data = const_cast*>(&test.initial); - options.flat_memory_base = test.flat_memory_base; - if (test.has_compute_info) { - options.wave_size = test.compute_info.wave_size; - } +CompiledShader CompileCase(const TestCase &test) { + auto user_data = + MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); + const auto uses_image = + std::any_of(test.opcodes.begin(), test.opcodes.end(), [](auto op) { + return op >= ShaderOpcode::ImageGetResinfo && + op <= ShaderOpcode::ImageGather4H; + }); + if (uses_image && ((user_data[3] >> 28u) & 0xfu) == 0) { + user_data[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; + } + if (uses_image) { + user_data[3] = (user_data[3] & ~0xfffu) | test.image_descriptor_swizzle; + } + if (!test.has_user_data) { + user_data[2] = static_cast(test.initial.size() * sizeof(u32)); + } + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = + test.has_compute_info ? &test.compute_info : nullptr; + options.user_data = user_data.data(); + options.read_memory = ReadTestMemory; + options.read_memory_data = const_cast *>(&test.initial); + options.flat_memory_base = test.flat_memory_base; + if (test.has_compute_info) { + options.wave_size = test.compute_info.wave_size; + } - ShaderRecompiler::CompileResult result; - std::string error; - if (!ShaderRecompiler::TryRecompile(test.code, options, result, &error)) { - Fail(test.name, "decode/IR", error.c_str()); - } - if (test.force_shader_data_storage) { - result.program.bindings = {}; - result.program.binding_layout_complete = false; - if (!ShaderRecompiler::IR::AllocateBindings(result.program, {.max_push_dwords = 0}, - &error)) { - Fail(test.name, "binding layout", error.c_str()); - } - const auto* shader_data = ShaderRecompiler::IR::FindBinding( - result.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData); - Require(test.name, "binding layout", - result.program.bindings.user_data_registers.empty() && - result.program.bindings.push_constant_size == 0 && shader_data != nullptr, - "offset-only shader data did not use its storage fallback"); - std::vector storage_spirv; - if (!ShaderRecompiler::Spirv::EmitProgram(result.program, result.resources, nullptr, - nullptr, options.compute_input_info, - storage_spirv, &error)) { - Fail(test.name, "SPIR-V emit", error.c_str()); - } - result.spirv = std::move(storage_spirv); - } - Require(test.name, "SPIR-V emit", !result.spirv.empty(), "recompiler returned empty SPIR-V"); - ValidateSpirv(test.name, result.spirv); - CheckSpirvText(test, result.spirv); - std::vector packed_user_data; - for (const auto reg: result.program.bindings.user_data_registers) { - packed_user_data.push_back(result.resources.user_data[reg - result.program.user_data_base]); - } - packed_user_data.resize(result.program.bindings.ShaderDataDwords()); - for (u32 i = 0; i < result.program.bindings.buffer_offset_count; i++) { - const auto offset = - i < test.storage_buffer_offsets.size() ? test.storage_buffer_offsets[i] : 0u; - Require(test.name, "shader data", offset % sizeof(u32) == 0 && offset < 256, - "storage buffer offset is not representable"); - packed_user_data[result.program.bindings.buffer_offset_dword + i / 4u] |= - offset << ((i % 4u) * 8u); - } - return {std::move(result.spirv), std::move(result.program), - std::move(result.resources.flattened_srt), std::move(packed_user_data)}; + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(test.code, options, result, &error)) { + Fail(test.name, "decode/IR", error.c_str()); + } + for (const auto &[text, expected] : test.decoded_counts) { + const auto actual = CountText(result.decoded_dump, text); + Require(test.name, "decoded RDNA2", + actual == expected, + text + " count=" + std::to_string(actual) + + ", expected=" + std::to_string(expected)); + } + for (const auto &[text, expected] : test.ir_counts) { + const auto actual = CountText(result.ir_dump, text); + Require(test.name, "typed IR", + actual == expected, + text + " count=" + std::to_string(actual) + + ", expected=" + std::to_string(expected)); + } + if (test.force_shader_data_storage) { + result.program.bindings = {}; + result.program.binding_layout_complete = false; + if (!ShaderRecompiler::IR::AllocateBindings( + result.program, {.max_push_dwords = 0}, &error)) { + Fail(test.name, "binding layout", error.c_str()); + } + const auto *shader_data = ShaderRecompiler::IR::FindBinding( + result.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::UserData); + Require(test.name, "binding layout", + result.program.bindings.user_data_registers.empty() && + result.program.bindings.push_constant_size == 0 && + shader_data != nullptr, + "offset-only shader data did not use its storage fallback"); + std::vector storage_spirv; + if (!ShaderRecompiler::Spirv::EmitProgram( + result.program, result.resources, nullptr, nullptr, + options.compute_input_info, storage_spirv, &error)) { + Fail(test.name, "SPIR-V emit", error.c_str()); + } + result.spirv = std::move(storage_spirv); + } + Require(test.name, "SPIR-V emit", !result.spirv.empty(), + "recompiler returned empty SPIR-V"); + ValidateSpirv(test.name, result.spirv); + CheckSpirvText(test, result.spirv); + std::vector packed_user_data; + for (const auto reg : result.program.bindings.user_data_registers) { + packed_user_data.push_back( + result.resources.user_data[reg - result.program.user_data_base]); + } + packed_user_data.resize(result.program.bindings.ShaderDataDwords()); + for (u32 i = 0; i < result.program.bindings.buffer_offset_count; i++) { + const auto offset = i < test.storage_buffer_offsets.size() + ? test.storage_buffer_offsets[i] + : 0u; + Require(test.name, "shader data", offset % sizeof(u32) == 0 && offset < 256, + "storage buffer offset is not representable"); + packed_user_data[result.program.bindings.buffer_offset_dword + i / 4u] |= + offset << ((i % 4u) * 8u); + } + return {std::move(result.spirv), std::move(result.program), + std::move(result.resources.flattened_srt), + std::move(packed_user_data)}; } -std::array MakeStructuredStorageBufferData(u32 stride_bytes, u32 num_records, - bool add_tid = false, u32 format = 0) { - std::array data {}; - data[1] = (stride_bytes & 0x3fffu) << 16u; - data[2] = num_records; - data[3] = 1u << 24u; - if (add_tid) { - data[3] |= 1u << 23u; - } - data[3] |= (format & 0x7fu) << 12u; - return data; +std::array MakeStructuredStorageBufferData(u32 stride_bytes, + u32 num_records, + bool add_tid = false, + u32 format = 0) { + std::array data{}; + data[1] = (stride_bytes & 0x3fffu) << 16u; + data[2] = num_records; + data[3] = 1u << 24u; + if (add_tid) { + data[3] |= 1u << 23u; + } + data[3] |= (format & 0x7fu) << 12u; + return data; } std::array MakeStorageTextureData(Prospero::BufferFormat format) { - std::array data {}; - data[0] = 0x1000u; - data[1] = (static_cast(format) & 0x1ffu) << 20u; - data[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; - return data; + std::array data{}; + data[0] = 0x1000u; + data[1] = (static_cast(format) & 0x1ffu) << 20u; + data[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; + return data; } -CompiledShader CompileFragmentCase(const GraphicsCase& test) { - const auto user_data = MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); - ShaderPixelInputInfo pixel_info {}; - pixel_info.input_num = test.pixel_interpolator_settings.empty() - ? 1u - : static_cast(test.pixel_interpolator_settings.size()); - pixel_info.ps_no_perspective = test.pixel_no_perspective; - for (u32 i = 0; i < std::size(pixel_info.interpolator_settings); i++) { - pixel_info.interpolator_settings[i] = i; - } - for (u32 i = 0; i < test.pixel_interpolator_settings.size() && - i < std::size(pixel_info.interpolator_settings); - i++) { - pixel_info.interpolator_settings[i] = test.pixel_interpolator_settings[i]; - } +CompiledShader CompileFragmentCase(const GraphicsCase &test) { + const auto user_data = + MakeNativeUserData(test.has_user_data ? &test.user_data : nullptr); + ShaderPixelInputInfo pixel_info{}; + pixel_info.input_num = + test.pixel_interpolator_settings.empty() + ? 1u + : static_cast(test.pixel_interpolator_settings.size()); + pixel_info.ps_no_perspective = test.pixel_no_perspective; + for (u32 i = 0; i < std::size(pixel_info.interpolator_settings); i++) { + pixel_info.interpolator_settings[i] = i; + } + for (u32 i = 0; i < test.pixel_interpolator_settings.size() && + i < std::size(pixel_info.interpolator_settings); + i++) { + pixel_info.interpolator_settings[i] = test.pixel_interpolator_settings[i]; + } - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.dump_ir = false; - options.pixel_input_info = &pixel_info; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = false; + options.pixel_input_info = &pixel_info; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - if (!ShaderRecompiler::TryRecompile(test.fragment_code, options, result, &error)) { - Fail(test.name, "decode/IR", error.c_str()); - } - Require(test.name, "SPIR-V emit", !result.spirv.empty(), "recompiler returned empty SPIR-V"); - ValidateSpirv(test.name, result.spirv); - std::vector packed_user_data; - for (const auto reg: result.program.bindings.user_data_registers) { - packed_user_data.push_back(result.resources.user_data[reg - result.program.user_data_base]); - } - packed_user_data.resize(result.program.bindings.ShaderDataDwords()); - return {std::move(result.spirv), std::move(result.program), - std::move(result.resources.flattened_srt), std::move(packed_user_data)}; + ShaderRecompiler::CompileResult result; + std::string error; + if (!ShaderRecompiler::TryRecompile(test.fragment_code, options, result, + &error)) { + Fail(test.name, "decode/IR", error.c_str()); + } + Require(test.name, "SPIR-V emit", !result.spirv.empty(), + "recompiler returned empty SPIR-V"); + ValidateSpirv(test.name, result.spirv); + std::vector packed_user_data; + for (const auto reg : result.program.bindings.user_data_registers) { + packed_user_data.push_back( + result.resources.user_data[reg - result.program.user_data_base]); + } + packed_user_data.resize(result.program.bindings.ShaderDataDwords()); + return {std::move(result.spirv), std::move(result.program), + std::move(result.resources.flattened_srt), + std::move(packed_user_data)}; } std::array MakeSampledTextureData(Prospero::BufferFormat format) { - return MakeStorageTextureData(format); + return MakeStorageTextureData(format); } namespace TestSpv { enum : u32 { - ExecutionModelVertex = 0, - AddressingModelLogical = 0, - MemoryModelGLSL450 = 1, - CapabilityShader = 1, - StorageClassInput = 1, - StorageClassOutput = 3, - FunctionControlNone = 0, - DecorationBlock = 2, - DecorationBuiltIn = 11, - DecorationLocation = 30, - BuiltInPosition = 0, - OpTypeVoid = 19, - OpTypeInt = 21, - OpTypeFloat = 22, - OpTypeVector = 23, - OpTypeStruct = 30, - OpTypePointer = 32, - OpTypeFunction = 33, - OpConstant = 43, - OpFunction = 54, - OpFunctionEnd = 56, - OpVariable = 59, - OpLoad = 61, - OpStore = 62, - OpAccessChain = 65, - OpDecorate = 71, - OpMemberDecorate = 72, - OpCompositeConstruct = 80, - OpCompositeExtract = 81, - OpLabel = 248, - OpReturn = 253, + ExecutionModelVertex = 0, + AddressingModelLogical = 0, + MemoryModelGLSL450 = 1, + CapabilityShader = 1, + StorageClassInput = 1, + StorageClassOutput = 3, + FunctionControlNone = 0, + DecorationBlock = 2, + DecorationBuiltIn = 11, + DecorationLocation = 30, + BuiltInPosition = 0, + OpTypeVoid = 19, + OpTypeInt = 21, + OpTypeFloat = 22, + OpTypeVector = 23, + OpTypeStruct = 30, + OpTypePointer = 32, + OpTypeFunction = 33, + OpConstant = 43, + OpFunction = 54, + OpFunctionEnd = 56, + OpVariable = 59, + OpLoad = 61, + OpStore = 62, + OpAccessChain = 65, + OpDecorate = 71, + OpMemberDecorate = 72, + OpCompositeConstruct = 80, + OpCompositeExtract = 81, + OpLabel = 248, + OpReturn = 253, }; std::vector MakePassthroughVertexSpirv() { - using ShaderRecompiler::Spirv::Builder; + using ShaderRecompiler::Spirv::Builder; - Builder b; - const auto void_type = b.AllocateId(); - const auto uint_type = b.AllocateId(); - const auto float_type = b.AllocateId(); - const auto vec2_type = b.AllocateId(); - const auto vec4_type = b.AllocateId(); - const auto per_vertex_type = b.AllocateId(); - const auto ptr_input_vec2 = b.AllocateId(); - const auto ptr_input_vec4 = b.AllocateId(); - const auto ptr_output_vec4 = b.AllocateId(); - const auto ptr_output_per_vertex = b.AllocateId(); - const auto func_type = b.AllocateId(); - const auto const_u32_0 = b.AllocateId(); - const auto const_f32_0 = b.AllocateId(); - const auto const_f32_1 = b.AllocateId(); - const auto in_pos = b.AllocateId(); - const auto in_color = b.AllocateId(); - const auto out_color = b.AllocateId(); - const auto per_vertex = b.AllocateId(); - const auto main = b.AllocateId(); - const auto label = b.AllocateId(); - const auto pos2 = b.AllocateId(); - const auto color4 = b.AllocateId(); - const auto pos_x = b.AllocateId(); - const auto pos_y = b.AllocateId(); - const auto position = b.AllocateId(); - const auto position_ptr = b.AllocateId(); + Builder b; + const auto void_type = b.AllocateId(); + const auto uint_type = b.AllocateId(); + const auto float_type = b.AllocateId(); + const auto vec2_type = b.AllocateId(); + const auto vec4_type = b.AllocateId(); + const auto per_vertex_type = b.AllocateId(); + const auto ptr_input_vec2 = b.AllocateId(); + const auto ptr_input_vec4 = b.AllocateId(); + const auto ptr_output_vec4 = b.AllocateId(); + const auto ptr_output_per_vertex = b.AllocateId(); + const auto func_type = b.AllocateId(); + const auto const_u32_0 = b.AllocateId(); + const auto const_f32_0 = b.AllocateId(); + const auto const_f32_1 = b.AllocateId(); + const auto in_pos = b.AllocateId(); + const auto in_color = b.AllocateId(); + const auto out_color = b.AllocateId(); + const auto per_vertex = b.AllocateId(); + const auto main = b.AllocateId(); + const auto label = b.AllocateId(); + const auto pos2 = b.AllocateId(); + const auto color4 = b.AllocateId(); + const auto pos_x = b.AllocateId(); + const auto pos_y = b.AllocateId(); + const auto position = b.AllocateId(); + const auto position_ptr = b.AllocateId(); - b.AddCapability({CapabilityShader}); - b.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); - b.AddEntryPoint(ExecutionModelVertex, main, "main", {in_pos, in_color, per_vertex, out_color}); - b.AddAnnotation({OpDecorate, in_pos, DecorationLocation, 0}); - b.AddAnnotation({OpDecorate, in_color, DecorationLocation, 1}); - b.AddAnnotation({OpDecorate, out_color, DecorationLocation, 0}); - b.AddAnnotation({OpDecorate, per_vertex_type, DecorationBlock}); - b.AddAnnotation({OpMemberDecorate, per_vertex_type, 0, DecorationBuiltIn, BuiltInPosition}); + b.AddCapability({CapabilityShader}); + b.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); + b.AddEntryPoint(ExecutionModelVertex, main, "main", + {in_pos, in_color, per_vertex, out_color}); + b.AddAnnotation({OpDecorate, in_pos, DecorationLocation, 0}); + b.AddAnnotation({OpDecorate, in_color, DecorationLocation, 1}); + b.AddAnnotation({OpDecorate, out_color, DecorationLocation, 0}); + b.AddAnnotation({OpDecorate, per_vertex_type, DecorationBlock}); + b.AddAnnotation({OpMemberDecorate, per_vertex_type, 0, DecorationBuiltIn, + BuiltInPosition}); - b.AddType({OpTypeVoid, void_type}); - b.AddType({OpTypeInt, uint_type, 32, 0}); - b.AddType({OpTypeFloat, float_type, 32}); - b.AddType({OpTypeVector, vec2_type, float_type, 2}); - b.AddType({OpTypeVector, vec4_type, float_type, 4}); - b.AddType({OpTypeStruct, per_vertex_type, vec4_type}); - b.AddType({OpTypePointer, ptr_input_vec2, StorageClassInput, vec2_type}); - b.AddType({OpTypePointer, ptr_input_vec4, StorageClassInput, vec4_type}); - b.AddType({OpTypePointer, ptr_output_vec4, StorageClassOutput, vec4_type}); - b.AddType({OpTypePointer, ptr_output_per_vertex, StorageClassOutput, per_vertex_type}); - b.AddType({OpTypeFunction, func_type, void_type}); - b.AddType({OpConstant, uint_type, const_u32_0, 0}); - b.AddType({OpConstant, float_type, const_f32_0, 0x00000000u}); - b.AddType({OpConstant, float_type, const_f32_1, 0x3f800000u}); - b.AddType({OpVariable, ptr_input_vec2, in_pos, StorageClassInput}); - b.AddType({OpVariable, ptr_input_vec4, in_color, StorageClassInput}); - b.AddType({OpVariable, ptr_output_vec4, out_color, StorageClassOutput}); - b.AddType({OpVariable, ptr_output_per_vertex, per_vertex, StorageClassOutput}); + b.AddType({OpTypeVoid, void_type}); + b.AddType({OpTypeInt, uint_type, 32, 0}); + b.AddType({OpTypeFloat, float_type, 32}); + b.AddType({OpTypeVector, vec2_type, float_type, 2}); + b.AddType({OpTypeVector, vec4_type, float_type, 4}); + b.AddType({OpTypeStruct, per_vertex_type, vec4_type}); + b.AddType({OpTypePointer, ptr_input_vec2, StorageClassInput, vec2_type}); + b.AddType({OpTypePointer, ptr_input_vec4, StorageClassInput, vec4_type}); + b.AddType({OpTypePointer, ptr_output_vec4, StorageClassOutput, vec4_type}); + b.AddType({OpTypePointer, ptr_output_per_vertex, StorageClassOutput, + per_vertex_type}); + b.AddType({OpTypeFunction, func_type, void_type}); + b.AddType({OpConstant, uint_type, const_u32_0, 0}); + b.AddType({OpConstant, float_type, const_f32_0, 0x00000000u}); + b.AddType({OpConstant, float_type, const_f32_1, 0x3f800000u}); + b.AddType({OpVariable, ptr_input_vec2, in_pos, StorageClassInput}); + b.AddType({OpVariable, ptr_input_vec4, in_color, StorageClassInput}); + b.AddType({OpVariable, ptr_output_vec4, out_color, StorageClassOutput}); + b.AddType( + {OpVariable, ptr_output_per_vertex, per_vertex, StorageClassOutput}); - b.AddFunction({OpFunction, void_type, main, FunctionControlNone, func_type}); - b.AddFunction({OpLabel, label}); - b.AddFunction({OpLoad, vec2_type, pos2, in_pos}); - b.AddFunction({OpLoad, vec4_type, color4, in_color}); - b.AddFunction({OpCompositeExtract, float_type, pos_x, pos2, 0}); - b.AddFunction({OpCompositeExtract, float_type, pos_y, pos2, 1}); - b.AddFunction( - {OpCompositeConstruct, vec4_type, position, pos_x, pos_y, const_f32_0, const_f32_1}); - b.AddFunction({OpAccessChain, ptr_output_vec4, position_ptr, per_vertex, const_u32_0}); - b.AddFunction({OpStore, position_ptr, position}); - b.AddFunction({OpStore, out_color, color4}); - b.AddFunction({OpReturn}); - b.AddFunction({OpFunctionEnd}); - return b.Build(); + b.AddFunction({OpFunction, void_type, main, FunctionControlNone, func_type}); + b.AddFunction({OpLabel, label}); + b.AddFunction({OpLoad, vec2_type, pos2, in_pos}); + b.AddFunction({OpLoad, vec4_type, color4, in_color}); + b.AddFunction({OpCompositeExtract, float_type, pos_x, pos2, 0}); + b.AddFunction({OpCompositeExtract, float_type, pos_y, pos2, 1}); + b.AddFunction({OpCompositeConstruct, vec4_type, position, pos_x, pos_y, + const_f32_0, const_f32_1}); + b.AddFunction( + {OpAccessChain, ptr_output_vec4, position_ptr, per_vertex, const_u32_0}); + b.AddFunction({OpStore, position_ptr, position}); + b.AddFunction({OpStore, out_color, color4}); + b.AddFunction({OpReturn}); + b.AddFunction({OpFunctionEnd}); + return b.Build(); } } // namespace TestSpv class VulkanHarness { public: - VulkanHarness() { Init(); } - ~VulkanHarness() { Destroy(); } - - VulkanHarness(const VulkanHarness&) = delete; - VulkanHarness& operator=(const VulkanHarness&) = delete; - - struct Buffer { - vk::Buffer buffer = nullptr; - vk::DeviceMemory memory = nullptr; - vk::DeviceSize size = 0; - bool coherent = false; - }; - - struct Image { - vk::Image image = nullptr; - vk::DeviceMemory memory = nullptr; - vk::ImageView view = nullptr; - vk::Format format = vk::Format::eUndefined; - vk::ImageLayout layout = vk::ImageLayout::eUndefined; - u32 width = 0; - u32 height = 0; - u32 layers = 1; - u32 mip_levels = 1; - u32 dwords_per_pixel = 0; - }; - - [[nodiscard]] vk::Device Device() const { return m_device; } - [[nodiscard]] GraphicContext& RuntimeContext() { - EnsureRuntimeContext(); - return m_runtime_context; - } - [[nodiscard]] RenderContext& RuntimeRenderer() { - EnsureRuntimeContext(); - return Renderer(); - } - - void CheckCommandPoolGrowth() { - EnsureRuntimeContext(); - std::vector> commands; - for (uint32_t i = 0; i < 12; i++) { - commands.push_back(std::make_unique(Renderer().GetCommandScheduler())); - } - for (auto& command: commands) { - Require("CommandPoolGrowth", "allocation", !command->IsInvalid(), - "unified command pool failed to grow"); - command->Begin(); - command->End(); - command->Execute(); - } - for (auto& command: commands) { - command->WaitForFence(); - } - std::printf("[host] %-32s ok\n", "CommandPoolGrowth"); - } - - void CheckSchedulerTimeline() { - EnsureRuntimeContext(); - CommandScheduler scheduler(Renderer(), m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - - const auto first_tick = scheduler.CurrentTick(); - uint32_t completed = 0; - scheduler.DeferOperation( - [owned = std::make_unique(1), &completed] { completed = *owned; }); - scheduler.Flush(); - scheduler.Wait(first_tick); - Require("SchedulerTimeline", "first tick", - scheduler.CurrentTick() == first_tick + 1 && scheduler.IsFree(first_tick) && - completed == 1, - "timeline wait did not release its deferred operation"); - - const auto second_tick = scheduler.CurrentTick(); - std::atomic priority_in_callback {false}; - scheduler.DeferPriorityOperation( - [owned = std::make_unique(2), &completed, &priority_in_callback] { - priority_in_callback = CommandScheduler::InDeferredOperation(); - completed = *owned; - }); - scheduler.Finish(); - scheduler.DrainPriorityOperations(); - Require("SchedulerTimeline", "finish", - scheduler.CurrentTick() == second_tick + 1 && scheduler.IsFree(second_tick) && - completed == 2 && priority_in_callback.load() && - !CommandScheduler::InDeferredOperation(), - "scheduler completion drain lost its tick or callback context"); - - const auto implicit_flush_tick = scheduler.CurrentTick(); - scheduler.DeferOperation( - [owned = std::make_unique(3), &completed] { completed = *owned; }); - scheduler.Wait(implicit_flush_tick); - Require("SchedulerTimeline", "implicit flush", - scheduler.CurrentTick() == implicit_flush_tick + 1 && - scheduler.IsFree(implicit_flush_tick) && completed == 3, - "waiting for the current tick did not flush it"); - - vk::SemaphoreTypeCreateInfo timeline_type {}; - timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo; - timeline_type.semaphoreType = vk::SemaphoreType::eTimeline; - vk::SemaphoreCreateInfo timeline_create {}; - timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo; - timeline_create.pNext = &timeline_type; - vk::Semaphore external_timeline = nullptr; - Require("SchedulerTimeline", "external timeline create", - m_runtime_context.device.createSemaphore( - &timeline_create, nullptr, &external_timeline) == vk::Result::eSuccess && - external_timeline != nullptr, - "failed to create the external timeline semaphore"); - - constexpr uint64_t external_wait_value = 9; - const auto external_wait_tick = scheduler.CurrentTick(); - SubmitInfo external_wait; - external_wait.AddWait(external_timeline, external_wait_value); - constexpr size_t blocked_submission_count = 6; - std::array blocked_handles {}; - for (size_t i = 0; i < blocked_submission_count; ++i) { - blocked_handles[i] = scheduler.Current().Handle(); - if (i == 0) { - scheduler.Flush(external_wait); - } else { - scheduler.Flush(); - } - } - bool distinct_handles = true; - for (size_t i = 0; i < blocked_handles.size(); ++i) { - for (size_t j = i + 1; j < blocked_handles.size(); ++j) { - distinct_handles &= blocked_handles[i] != blocked_handles[j]; - } - } - const auto last_blocked_tick = scheduler.CurrentTick() - 1; - Require("SchedulerTimeline", "external timeline overflow", - distinct_handles && last_blocked_tick == - external_wait_tick + blocked_submission_count - 1 && - !scheduler.IsFree(last_blocked_tick), - "blocked submissions did not grow the command-buffer pool"); - vk::SemaphoreSignalInfo signal_info {}; - signal_info.sType = vk::StructureType::eSemaphoreSignalInfo; - signal_info.semaphore = external_timeline; - signal_info.value = external_wait_value; - Require("SchedulerTimeline", "external timeline signal", - m_runtime_context.device.signalSemaphore(&signal_info) == vk::Result::eSuccess, - "failed to signal the external timeline semaphore"); - scheduler.Wait(last_blocked_tick); - m_runtime_context.device.destroySemaphore(external_timeline, nullptr); - scheduler.Flush(); - scheduler.Flush(); - Require("SchedulerTimeline", "timeline pool reuse", - std::ranges::find(blocked_handles, scheduler.Current().Handle()) != - blocked_handles.end(), - "completed command buffers were not reused after timeline progress"); - scheduler.Shutdown(); - - CommandScheduler draining(Renderer(), m_runtime_context); - HW::Context drain_registers {}; - HW::UserConfig drain_user_config {}; - HW::Shader drain_shaders {}; - draining.Begin(drain_registers, drain_user_config, drain_shaders); - uint32_t reentrant = 0; - draining.DeferOperation([&draining, &reentrant] { - reentrant++; - draining.DeferOperation([&reentrant] { reentrant++; }); - }); - - std::binary_semaphore priority_entered {0}; - std::binary_semaphore release_priority {0}; - draining.DeferPriorityOperation([&draining, &priority_entered, &release_priority] { - priority_entered.release(); - draining.Shutdown(); - release_priority.acquire(); - }); - std::jthread shutdown_thread([&draining] { draining.Shutdown(); }); - priority_entered.acquire(); - std::jthread second_shutdown_thread([&draining] { draining.Shutdown(); }); - - std::atomic concurrent_completed {false}; - std::jthread concurrent_defer([&draining, &concurrent_completed] { - draining.DeferOperation([&concurrent_completed] { concurrent_completed = true; }); - }); - release_priority.release(); - shutdown_thread.join(); - second_shutdown_thread.join(); - concurrent_defer.join(); - Require("SchedulerTimeline", "shutdown drain", - reentrant == 2 && concurrent_completed.load(), - "shutdown lost reentrant or concurrent deferred work"); - std::printf("[host] %-32s ok\n", "SchedulerTimeline"); - } - - void CheckGpuMappedRangeLifecycle() { - EnsureRuntimeContext(); - auto& context = Renderer(); - CommandScheduler scheduler(context, m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - context.InitializeGpu(nullptr); - auto& gpu = context.GetGpu(); - GpuResourceManager resources(m_runtime_context, scheduler); - resources.SetGpu(&gpu); - - constexpr uint64_t base = 0x0000000200000000ull; - constexpr uint64_t page = 0x4000; - resources.MapMemory(base, page * 4); - resources.MapMemory(base + page * 2, page * 4); - Require("GpuMappedRangeLifecycle", "union", - resources.IsMapped(base, page * 6) && !resources.IsMapped(base, page * 7), - "overlapping maps did not form one interval union"); - - resources.UnmapMemory(base + page * 2, page * 2); - Require("GpuMappedRangeLifecycle", "subtract", - resources.IsMapped(base, page * 2) && - resources.IsMapped(base + page * 4, page * 2) && - !resources.IsMapped(base, page * 6), - "partial unmap did not punch the expected interval hole"); - - resources.UnmapMemory(base + page * 2, page * 2); - Require("GpuMappedRangeLifecycle", "idempotent unmap", - resources.IsMapped(base, page * 2) && resources.IsMapped(base + page * 4, page * 2), - "unmapping an absent interval changed neighboring mappings"); - - resources.UnmapMemory(base, page * 6); - Require("GpuMappedRangeLifecycle", "clear", !resources.IsMapped(base, page * 6), - "full unmap did not clear the interval union"); - - constexpr uint64_t old_prt = base + page * 8; - constexpr uint64_t new_prt = base + page * 16; - resources.MapMemory(old_prt, page * 4); - resources.UnmapMemory(old_prt, page * 4); - resources.MapMemory(new_prt, page * 6); - Require("GpuMappedRangeLifecycle", "PRT replacement", - !resources.IsMapped(old_prt, page * 4) && resources.IsMapped(new_prt, page * 6), - "old-unmap/new-map did not replace full PRT coverage"); - - resources.SetGpu(nullptr); - scheduler.Finish(); - context.ShutdownGpu(); - std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle"); - } - - void CheckStreamBufferRing() { - EnsureRuntimeContext(); - CommandScheduler scheduler(Renderer(), m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - GpuResourceManager resources(m_runtime_context, scheduler); - auto& cache = resources.GetBufferCache(); - constexpr std::array, 4> utilities {{ - {MemoryUsage::Upload, 512ull << 20}, - {MemoryUsage::Stream, 64ull << 20}, - {MemoryUsage::Download, 32ull << 20}, - {MemoryUsage::DeviceLocal, 128ull << 20}, - }}; - std::array handles {}; - for (size_t i = 0; i < utilities.size(); i++) { - auto& utility = cache.GetUtilityBuffer(utilities[i].first); - handles[i] = utility.Handle(); - Require("StreamBufferRing", "utility ownership", - utility.Usage() == utilities[i].first && - utility.Size() == utilities[i].second && utility.Handle() != nullptr, - "BufferCache utility buffer does not match the fixed layout"); - } - for (size_t i = 0; i < handles.size(); i++) { - for (size_t j = i + 1; j < handles.size(); j++) { - Require("StreamBufferRing", "utility identity", handles[i] != handles[j], - "utility usages alias the same native buffer"); - } - } - auto* fixed_download = &cache.GetUtilityBuffer(MemoryUsage::Download); - const auto fixed_handle = fixed_download->Handle(); - const auto [oversized_download, oversized_offset] = - fixed_download->Map(fixed_download->Size() + 4, 64, false); - const auto [download_probe, download_probe_offset] = fixed_download->Map(64, 64, false); - Require("StreamBufferRing", "fixed download utility", - oversized_download == nullptr && oversized_offset == 0 && - download_probe != nullptr && download_probe_offset == 0 && - &cache.GetUtilityBuffer(MemoryUsage::Download) == fixed_download && - fixed_download->Size() == (32ull << 20) && - fixed_download->Handle() == fixed_handle, - "oversized download replaced or corrupted the fixed shared ring"); - fixed_download->Commit(); - - std::vector full_stream(64ull << 20, 0x5a); - const auto utility_tick = scheduler.CurrentTick(); - auto full_binding = cache.UploadTransient(full_stream.data(), full_stream.size(), 16); - const std::array overflow_data {0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, - 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, - 0xa5, 0xa5, 0xa5, 0xa5}; - auto overflow_binding = - cache.UploadTransient(overflow_data.data(), overflow_data.size(), 16); - auto overflow_owner = - std::static_pointer_cast(overflow_binding.owner); - Require("StreamBufferRing", "current tick overflow", - full_binding.owner == nullptr && - full_binding.buffer == cache.GetUtilityBuffer(MemoryUsage::Stream).Handle() && - overflow_owner != nullptr && - overflow_binding.buffer == overflow_owner->Handle() && - overflow_binding.buffer != full_binding.buffer && - overflow_owner->Mapped().size() == overflow_data.size() && - std::memcmp(overflow_owner->Mapped().data(), overflow_data.data(), - overflow_data.size()) == 0 && - scheduler.CurrentTick() == utility_tick, - "current-tick stream overflow submitted partial state or lost bytes"); - - const auto atom = m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize; - uint64_t policy_size = 17; - uint64_t policy_alignment = 12; - uint64_t coherent_size = policy_size; - uint64_t coherent_alignment = policy_alignment; - Require("StreamBufferRing", "non-coherent reservation policy", - StreamBufferTestAccess::NormalizeReservation(false, 256, policy_size, - policy_alignment) && - policy_size == 256 && policy_alignment == 768 && - StreamBufferTestAccess::NormalizeReservation(true, 256, coherent_size, - coherent_alignment) && - coherent_size == 17 && coherent_alignment == 12, - "non-coherent reservations are not isolated at the atom/texel LCM"); - const auto small_ring_size = std::max(64, atom * 2); - StreamBuffer stream(m_runtime_context, scheduler, MemoryUsage::Upload, small_ring_size); - const auto first_size = - stream.IsCoherent() ? small_ring_size - 16 : small_ring_size - atom + 1; - constexpr uint64_t blocked_size = 24; - const auto [first, first_offset] = stream.Map(first_size, 16, false); - Require("StreamBufferRing", "first allocation", first != nullptr && first_offset == 0, - "initial stream allocation failed"); - std::memset(first, 0x5a, static_cast(first_size)); - stream.Commit(); - - const auto [oversize, oversize_offset] = stream.Map(small_ring_size + 1, 1, false); - Require("StreamBufferRing", "oversize", oversize == nullptr && oversize_offset == 0, - "oversize allocation was accepted"); - const auto [blocked, blocked_offset] = stream.Map(blocked_size, 1, false); - const auto [blocked_again, blocked_again_offset] = stream.Map(blocked_size, 1, false); - Require("StreamBufferRing", "transactional failure", - blocked == nullptr && blocked_offset == 0 && blocked_again == nullptr && - blocked_again_offset == 0, - "failed wrap mutated stream state"); - - scheduler.Flush(); - const auto [wrapped, wrapped_offset] = stream.Map(blocked_size, 1); - Require("StreamBufferRing", "watched wrap", wrapped != nullptr && wrapped_offset == 0, - "completed watch did not release the wrapped allocation"); - stream.Commit(); - - const auto download_alignment = atom / std::gcd(atom, 12) * 12; - const auto download_ring_size = - std::max(64, download_alignment + std::max(atom, 16)); - StreamBuffer download(m_runtime_context, scheduler, MemoryUsage::Download, - download_ring_size); - const auto [download_data, download_offset] = download.Map(16, 8); - Require("StreamBufferRing", "download allocation", - download_data != nullptr && download_offset == 0, - "download stream is not host visible"); - download.Commit(); - scheduler.Flush(); - const auto [second_download, second_download_offset] = download.Map(16, 12); - Require("StreamBufferRing", "download atom isolation", - second_download != nullptr && - (download.IsCoherent() || - (download_offset % atom == 0 && second_download_offset % atom == 0 && - second_download_offset >= download_offset + atom)) && - second_download_offset % 12 == 0, - "download reservations share a non-coherent atom or lost caller " - "alignment"); - download.Commit(); - - scheduler.Finish(); - download.Invalidate(download_offset, 16); - std::printf("[host] %-32s ok\n", "StreamBufferRing"); - } - - void CheckGpuCommandLane() { - EnsureRuntimeContext(); - auto& context = Renderer(); - context.InitializeGpu(nullptr); - auto& gpu = context.GetGpu(); - - const auto caller_thread = std::this_thread::get_id(); - std::thread::id gpu_thread; - std::thread::id nested_thread; - uint32_t order = 0; - gpu.SendCommand([&order] { order = 1; }); - gpu.SendCommandSync([&] { - gpu_thread = std::this_thread::get_id(); - Require("GpuCommandLane", "FIFO", order == 1, - "synchronous command overtook an older host command"); - Require("GpuCommandLane", "GPU context", Gpu::IsGpuThread(), - "host command did not run on the GPU thread"); - gpu.SendCommandSync([&nested_thread] { nested_thread = std::this_thread::get_id(); }); - order = 2; - }); - Require("GpuCommandLane", "dispatch thread", - order == 2 && gpu_thread != caller_thread && nested_thread == gpu_thread, - "host command did not run on the GPU thread or nested sync deadlocked"); - - { - const auto make_release_mem = [](uint32_t data_sel, uint32_t interrupt_selector, - void* destination, uint64_t value, - uint32_t action = 0x28u, - uint32_t gcr_cntl = 1u << 9u) { - std::array packet {}; - const auto address = reinterpret_cast(destination); - const auto event_index = (action >= 0x2fu ? 6u : 5u); - packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM); - packet[1] = action | (event_index << 8u) | (gcr_cntl << 12u); - packet[2] = (interrupt_selector << 24u) | (data_sel << 29u); - packet[3] = static_cast(address); - packet[4] = static_cast(address >> 32u); - packet[5] = static_cast(value); - packet[6] = static_cast(value >> 32u); - packet[7] = 0x123u; - return packet; - }; - - vk::SemaphoreTypeCreateInfo timeline_type {}; - timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo; - timeline_type.semaphoreType = vk::SemaphoreType::eTimeline; - vk::SemaphoreCreateInfo timeline_create {}; - timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo; - timeline_create.pNext = &timeline_type; - vk::Semaphore blocked_timeline = nullptr; - Require("GpuCommandLane", "blocked timeline create", - m_runtime_context.device.createSemaphore( - &timeline_create, nullptr, &blocked_timeline) == vk::Result::eSuccess && - blocked_timeline != nullptr, - "failed to create the blocked GPU timeline"); - - constexpr uint64_t release_tick = 1; - auto& gpu_scheduler = context.GetCommandScheduler(); - auto processor = std::make_unique(context); - gpu.SendCommandSync([&] { - processor->BufferInit(); - SubmitInfo blocked_submit; - blocked_submit.AddWait(blocked_timeline, release_tick); - gpu_scheduler.Flush(blocked_submit); - }); - - std::binary_semaphore parser_complete {0}; - alignas(uint64_t) uint64_t interrupt_only_gds_label = UINT64_MAX; - alignas(uint64_t) uint64_t cb_db_release_label = UINT64_MAX; - bool parser_kept_nonblocking_boundaries = false; - std::jthread no_gpu_wait([&] { - gpu.SendCommandSync([&] { - processor->BufferInit(); - const auto tick_before = gpu_scheduler.CurrentTick(); - processor->TriggerEvent(0x16u, 0u); - auto packet = make_release_mem(0, 0, nullptr, 0); - Pm4Execution execution; - const auto result = processor->Process(execution, packet.data(), packet.size()); - processor->IncrementDe(); - const bool cache_and_counter_did_not_submit = - result == Pm4ProcessResult::Complete && - gpu_scheduler.CurrentTick() == tick_before; - - const auto cb_db_tick = gpu_scheduler.CurrentTick(); - auto cb_db_release = make_release_mem(2, 0, &cb_db_release_label, 0, 0x14u, 0); - Pm4Execution cb_db_execution; - const auto cb_db_result = processor->Process( - cb_db_execution, cb_db_release.data(), cb_db_release.size()); - const bool cb_db_release_did_not_submit = - cb_db_result == Pm4ProcessResult::Complete && cb_db_release_label == 0 && - gpu_scheduler.CurrentTick() == cb_db_tick; - - auto gds_interrupt_only = - make_release_mem(5, 1, &interrupt_only_gds_label, 1ull << 16u); - Pm4Execution gds_interrupt_execution; - const auto interrupt_tick = gpu_scheduler.CurrentTick(); - const auto interrupt_result = processor->Process( - gds_interrupt_execution, gds_interrupt_only.data(), gds_interrupt_only.size()); - parser_kept_nonblocking_boundaries = - cache_and_counter_did_not_submit && cb_db_release_did_not_submit && - interrupt_result == Pm4ProcessResult::Complete && - gpu_scheduler.CurrentTick() == interrupt_tick + 1 && - interrupt_only_gds_label == UINT64_MAX; - }); - parser_complete.release(); - }); - const bool parser_did_not_wait_gpu = - parser_complete.try_acquire_for(std::chrono::seconds(2)); - - vk::SemaphoreSignalInfo signal_info {}; - signal_info.sType = vk::StructureType::eSemaphoreSignalInfo; - signal_info.semaphore = blocked_timeline; - signal_info.value = release_tick; - Require("GpuCommandLane", "blocked timeline signal", - m_runtime_context.device.signalSemaphore(&signal_info) == vk::Result::eSuccess, - "failed to release the blocked GPU timeline"); - no_gpu_wait.join(); - gpu.SendCommandSync([&] { gpu_scheduler.Finish(); }); - m_runtime_context.device.destroySemaphore(blocked_timeline, nullptr); - Require("GpuCommandLane", "nonblocking GPU packets", - parser_did_not_wait_gpu && parser_kept_nonblocking_boundaries, - "a cache event, GL2 writeback, DE counter, or interrupt boundary used the wrong " - "submission behavior"); - - alignas(uint64_t) uint64_t release_label = 0; - alignas(uint64_t) uint64_t gds_label = UINT64_MAX; - bool release_mem_submission_counts = false; - gpu.SendCommandSync([&] { - processor->BufferInit(); - - auto immediate = make_release_mem(1, 0, &release_label, 0x11223344u); - Pm4Execution immediate_execution; - const auto immediate_tick = gpu_scheduler.CurrentTick(); - const auto immediate_result = - processor->Process(immediate_execution, immediate.data(), immediate.size()); - const bool immediate_split_once = - gpu_scheduler.CurrentTick() == immediate_tick + 1; - - auto gds = make_release_mem(5, 0, &gds_label, 1ull << 16u); - Pm4Execution gds_execution; - const auto gds_tick = gpu_scheduler.CurrentTick(); - const auto gds_result = processor->Process(gds_execution, gds.data(), gds.size()); - const bool gds_waited_once = gpu_scheduler.CurrentTick() == gds_tick + 1; - - auto interrupt_only = make_release_mem(0, 4, nullptr, 0); - Pm4Execution interrupt_execution; - const auto interrupt_tick = gpu_scheduler.CurrentTick(); - const auto interrupt_result = processor->Process( - interrupt_execution, interrupt_only.data(), interrupt_only.size()); - const bool interrupt_split_once = - gpu_scheduler.CurrentTick() == interrupt_tick + 1; - - auto gds_interrupt = make_release_mem(5, 2, &gds_label, 1ull << 16u); - Pm4Execution gds_interrupt_execution; - const auto gds_interrupt_tick = gpu_scheduler.CurrentTick(); - const auto gds_interrupt_result = processor->Process( - gds_interrupt_execution, gds_interrupt.data(), gds_interrupt.size()); - const bool gds_interrupt_waited_once = - gpu_scheduler.CurrentTick() == gds_interrupt_tick + 1; - - release_mem_submission_counts = - immediate_result == Pm4ProcessResult::Complete && immediate_split_once && - gds_result == Pm4ProcessResult::Complete && gds_waited_once && - interrupt_result == Pm4ProcessResult::Complete && interrupt_split_once && - gds_interrupt_result == Pm4ProcessResult::Complete && - gds_interrupt_waited_once; - }); - gpu.SendCommandSync([&] { gpu_scheduler.Finish(); }); - Require("GpuCommandLane", "RELEASE_MEM submission counts", - release_mem_submission_counts && - static_cast(release_label) == 0x11223344u && - static_cast(gds_label) == 0, - "RELEASE_MEM lost its required split/readback or retained a redundant GPU wait"); - } - - alignas(uint32_t) uint32_t packet_marker_a = 0; - alignas(uint32_t) uint32_t packet_marker_b = 0; - const auto write_packet = [](uint32_t* packet, uint32_t* destination, uint32_t value) { - const auto destination_address = reinterpret_cast(destination); - packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - packet[1] = 0; - packet[2] = static_cast(destination_address); - packet[3] = static_cast(destination_address >> 32u); - packet[4] = value; - }; - std::array polling_loop {}; - for (size_t i = 0; i < polling_loop.size() - 4; i += 2) { - polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); - polling_loop[i + 1] = 0; - } - const auto loop_address = reinterpret_cast(polling_loop.data()); - polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); - polling_loop[1021] = static_cast(loop_address); - polling_loop[1022] = static_cast(loop_address >> 32u); - polling_loop[1023] = 0x0f200000u | static_cast(polling_loop.size()); - - std::array polling_commands {}; - write_packet(polling_commands.data(), &packet_marker_a, 11); - polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); - polling_commands[6] = static_cast(loop_address); - polling_commands[7] = static_cast(loop_address >> 32u); - polling_commands[8] = 0x0f200000u | static_cast(polling_loop.size()); - write_packet(polling_commands.data() + 9, &packet_marker_b, 22); - std::binary_semaphore stream_gate_entered {0}; - std::binary_semaphore stream_gate_release {0}; - gpu.SendCommand([&] { - stream_gate_entered.release(); - stream_gate_release.acquire(); - }); - gpu.Submit(polling_commands.data(), polling_commands.size(), nullptr, 0); - stream_gate_entered.acquire(); - stream_gate_release.release(); - uint32_t packet_marker_a_at_callback = UINT32_MAX; - uint32_t packet_marker_b_at_callback = UINT32_MAX; - for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11; attempt++) { - gpu.SendCommandSync([&] { - if (packet_marker_a == 11 && packet_marker_b == 0) { - packet_marker_a_at_callback = packet_marker_a; - packet_marker_b_at_callback = packet_marker_b; - polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); - polling_loop[1021] = 0; - polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); - polling_loop[1023] = 0; - } - }); - std::this_thread::yield(); - } - gpu.Done(); - Require("GpuCommandLane", "packet-boundary command polling", - packet_marker_a_at_callback == 11 && packet_marker_b_at_callback == 0 && - packet_marker_a == 11 && packet_marker_b == 22, - "host command waited for an entire non-suspended PM4 stream"); - - uint32_t label = 0; - uint32_t prefix = 0; - uint32_t suffix = 0; - const auto address = [](const void* value) { return reinterpret_cast(value); }; - std::array commands {}; - commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - commands[1] = 0; - commands[2] = static_cast(address(&prefix)); - commands[3] = static_cast(address(&prefix) >> 32u); - commands[4] = 11; - commands[5] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); - commands[6] = static_cast(address(&label)); - commands[7] = static_cast(address(&label) >> 32u); - commands[8] = UINT32_MAX; - commands[9] = 1; - commands[10] = 0x10u | 3u; - commands[12] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - commands[13] = 0; - commands[14] = static_cast(address(&suffix)); - commands[15] = static_cast(address(&suffix) >> 32u); - commands[16] = 22; - gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, 0); - - std::binary_semaphore ordered_started {0}; - std::atomic ordered_finished {false}; - uint32_t ordered_suffix = 0; - std::jthread ordered([&] { - ordered_started.release(); - gpu.Done(); - ordered_suffix = suffix; - ordered_finished = true; - }); - ordered_started.acquire(); - gpu.SendCommandSync([&] { - Require("GpuCommandLane", "submit done barrier", !ordered_finished.load(), - "submit done returned before a queued submission"); - label = 1; - }); - ordered.join(); - Require("GpuCommandLane", "ordered completion", - prefix == 11 && suffix == 22 && ordered_suffix == 22 && ordered_finished.load(), - "submit done did not drain prior PM4 work"); - - auto& resources = context.GetGpuResources(); - constexpr uint64_t empty_unmap_base = 0x0000000200400000ull; - constexpr uint64_t empty_unmap_size = 0x4000; - resources.MapMemory(empty_unmap_base, empty_unmap_size); - label = 0; - prefix = 0; - suffix = 0; - gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, 0); - - std::binary_semaphore unmap_complete {0}; - std::jthread unmap_thread([&] { - resources.UnmapMemory(empty_unmap_base, empty_unmap_size); - unmap_complete.release(); - }); - const bool unmap_returned = unmap_complete.try_acquire_for(std::chrono::seconds(2)); - gpu.SendCommandSync([&] { label = 1; }); - if (!unmap_returned) { - unmap_complete.acquire(); - } - unmap_thread.join(); - gpu.Done(); - Require("GpuCommandLane", "unmap queue progress", - unmap_returned && !resources.IsMapped(empty_unmap_base, empty_unmap_size) && - prefix == 11 && suffix == 22, - "an unrelated unmap waited for a blocked PM4 submission"); - - auto& scheduler = context.GetCommandScheduler(); - std::atomic normal_completed {false}; - gpu.SendCommandSync( - [&] { scheduler.DeferOperation([&] { normal_completed = true; }); }); - resources.MapMemory(empty_unmap_base, empty_unmap_size); - resources.UnmapMemory(empty_unmap_base, empty_unmap_size); - Require("GpuCommandLane", "unmap native completion", - normal_completed.load() && - !resources.IsMapped(empty_unmap_base, empty_unmap_size), - "unmap returned before an earlier native guest-memory callback"); - - std::binary_semaphore priority_entered {0}; - std::binary_semaphore release_priority {0}; - gpu.SendCommandSync([&] { - scheduler.DeferPriorityOperation([&] { - priority_entered.release(); - release_priority.acquire(); - }); - scheduler.Flush(); - }); - priority_entered.acquire(); - resources.MapMemory(empty_unmap_base, empty_unmap_size); - - std::binary_semaphore priority_unmap_entered {0}; - std::binary_semaphore priority_unmap_complete {0}; - std::jthread priority_unmap_thread([&] { - gpu.SendCommandSync([&] { - priority_unmap_entered.release(); - resources.UnmapMemory(empty_unmap_base, empty_unmap_size); - }); - priority_unmap_complete.release(); - }); - priority_unmap_entered.acquire(); - const bool unmap_overtook_priority = - priority_unmap_complete.try_acquire_for(std::chrono::seconds(1)); - release_priority.release(); - if (!unmap_overtook_priority) { - priority_unmap_complete.acquire(); - } - priority_unmap_thread.join(); - Require("GpuCommandLane", "unmap priority ordering", - !unmap_overtook_priority && - !resources.IsMapped(empty_unmap_base, empty_unmap_size), - "unmap returned before an earlier guest-memory callback"); - - constexpr uintptr_t fault_base = 0x0000000200500000ull; - constexpr uint64_t fault_size = 0x10000; - int64_t fault_direct_offset = -1; - Require("GpuCommandLane", "processor fault direct allocation", - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), fault_size, fault_size, - 0, &fault_direct_offset) == 0, - "processor-fault direct-memory allocation failed"); - void* fault_memory = reinterpret_cast(fault_base); - Require("GpuCommandLane", "processor fault direct mapping", - Libs::LibKernel::Memory::KernelMapDirectMemory( - &fault_memory, fault_size, 0x3, 0x10, fault_direct_offset, fault_size) == 0, - "processor-fault fixed direct-memory mapping failed"); - Require("GpuCommandLane", "processor fault allocation", - fault_memory == reinterpret_cast(fault_base), - "fixed processor-fault allocation failed"); - resources.MapMemory(fault_base, fault_size); - - constexpr uint64_t immediate_dst = fault_base + 0x1000; - constexpr uint64_t immediate_memory_dst = fault_base + 0x2000; - constexpr uint64_t memory_src = fault_base + 0x4000; - constexpr uint64_t memory_copy_dst = fault_base + 0x6000; - constexpr uint64_t memory_dst = fault_base + 0x8000; - constexpr uint64_t l2_copy_dst = fault_base + 0xa000; - constexpr uint64_t clean_cached_fill = fault_base + 0x3000; - constexpr uint64_t clean_cached_copy = fault_base + 0x3008; - constexpr uint64_t clean_cached_source = fault_base + 0x3010; - constexpr uint64_t clean_cached_readback = fault_base + 0xb000; - constexpr std::array source_words {0x11223344u, 0x55667788u}; - constexpr std::array clean_copy_words {0x13579bdfu, 0x2468ace0u}; - std::memcpy(reinterpret_cast(memory_src), source_words.data(), sizeof(source_words)); - std::memcpy(reinterpret_cast(clean_cached_source), clean_copy_words.data(), - sizeof(clean_copy_words)); - - std::array dma_commands {}; - size_t dma_cursor = 0; - const auto append_dma = [&](uint8_t dst_sel, uint64_t dst, uint8_t src_sel, uint64_t src, - uint32_t bytes) { - auto* packet = dma_commands.data() + dma_cursor; - dma_cursor += 7; - packet[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0); - packet[1] = - (static_cast(dst_sel) << 20u) | (static_cast(src_sel) << 29u); - packet[2] = static_cast(src); - packet[3] = static_cast(src >> 32u); - packet[4] = static_cast(dst); - packet[5] = static_cast(dst >> 32u); - packet[6] = bytes; - }; - constexpr uint64_t gds_immediate_offset = 0; - constexpr uint64_t gds_memory_offset = 0x10; - constexpr uint32_t immediate_value = 0xa1b2c3d4u; - append_dma(1, gds_immediate_offset, 2, immediate_value, sizeof(source_words)); - append_dma(0, immediate_dst, 1, gds_immediate_offset, sizeof(source_words)); - append_dma(1, gds_memory_offset, 0, memory_src, sizeof(source_words)); - append_dma(0, memory_dst, 1, gds_memory_offset, sizeof(source_words)); - append_dma(0, immediate_memory_dst, 2, immediate_value, sizeof(source_words)); - append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words)); - append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words)); - Require("GpuCommandLane", "DMA_DATA packet assembly", dma_cursor == dma_commands.size(), - "DMA_DATA GDS packet stream has the wrong size"); - gpu.Submit(dma_commands.data(), static_cast(dma_commands.size()), nullptr, 0); - gpu.Done(); - constexpr uint32_t clean_fill_value = 0xdecafbad; - gpu.SendCommandSync([&] { - auto& buffer_cache = resources.GetBufferCache(); - Require("GpuCommandLane", "clean cached setup", - buffer_cache.IsRegionRegistered(clean_cached_fill, sizeof(source_words)) && - buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(source_words)), - "DMA did not establish a cached page with a dirty sibling"); - - buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), clean_fill_value); - buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, - sizeof(clean_copy_words)); - Require( - "GpuCommandLane", "clean cached DMA ownership", - !buffer_cache.HasGpuDirtyBytes(clean_cached_fill, sizeof(source_words)) && - !buffer_cache.HasGpuDirtyBytes(clean_cached_copy, sizeof(clean_copy_words)) && - buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(source_words)), - "clean cached DMA acquired ownership or released a byte-disjoint " - "dirty sibling"); - - constexpr uint64_t gds_clean_mirror = 0x20; - buffer_cache.CopyBuffer(gds_clean_mirror, clean_cached_fill, 2 * sizeof(source_words), - true, false); - buffer_cache.CopyBuffer(clean_cached_readback, gds_clean_mirror, - 2 * sizeof(source_words), false, true); - }); - Require("GpuCommandLane", "DMA_DATA immediate GDS readback", - resources.HandleFault(PageFaultAccess::Read, immediate_dst), - "immediate-to-GDS-to-memory copy did not publish GPU bytes"); - Require("GpuCommandLane", "DMA_DATA memory GDS readback", - resources.HandleFault(PageFaultAccess::Read, memory_dst), - "memory-to-GDS-to-memory copy did not publish GPU bytes"); - Require("GpuCommandLane", "DMA_DATA immediate memory readback", - resources.HandleFault(PageFaultAccess::Read, immediate_memory_dst), - "immediate-to-memory copy did not publish GPU bytes"); - Require("GpuCommandLane", "DMA_DATA memory copy readback", - resources.HandleFault(PageFaultAccess::Read, memory_copy_dst), - "memory-to-memory copy did not publish GPU bytes"); - Require("GpuCommandLane", "DMA_DATA L2 readback", - resources.HandleFault(PageFaultAccess::Read, l2_copy_dst), - "MemoryUsingL2 copy did not publish GPU bytes"); - Require("GpuCommandLane", "clean cached mirror readback", - resources.HandleFault(PageFaultAccess::Read, clean_cached_readback), - "clean host DMA was not reflected in the cached buffer"); - std::array immediate_words {}; - std::array copied_words {}; - std::array immediate_memory_words {}; - std::array memory_copy_words {}; - std::array l2_copy_words {}; - std::array clean_cached_words {}; - std::memcpy(immediate_words.data(), reinterpret_cast(immediate_dst), - sizeof(immediate_words)); - std::memcpy(copied_words.data(), reinterpret_cast(memory_dst), - sizeof(copied_words)); - std::memcpy(immediate_memory_words.data(), - reinterpret_cast(immediate_memory_dst), - sizeof(immediate_memory_words)); - std::memcpy(memory_copy_words.data(), reinterpret_cast(memory_copy_dst), - sizeof(memory_copy_words)); - std::memcpy(l2_copy_words.data(), reinterpret_cast(l2_copy_dst), - sizeof(l2_copy_words)); - std::memcpy(clean_cached_words.data(), reinterpret_cast(clean_cached_readback), - sizeof(clean_cached_words)); - Require("GpuCommandLane", "DMA_DATA immediate GDS contents", - immediate_words == std::array {immediate_value, immediate_value}, - "immediate-to-GDS-to-memory bytes do not match"); - Require("GpuCommandLane", "DMA_DATA memory GDS contents", copied_words == source_words, - "memory-to-GDS-to-memory bytes do not match"); - Require("GpuCommandLane", "DMA_DATA immediate memory contents", - immediate_memory_words == - std::array {immediate_value, immediate_value}, - "immediate-to-memory bytes do not match"); - Require("GpuCommandLane", "DMA_DATA memory copy contents", - memory_copy_words == source_words, "memory-to-memory bytes do not match"); - Require("GpuCommandLane", "DMA_DATA L2 contents", l2_copy_words == source_words, - "MemoryUsingL2 bytes do not match"); - Require("GpuCommandLane", "clean cached DMA contents", - clean_cached_words == std::array {clean_fill_value, clean_fill_value, - clean_copy_words[0], - clean_copy_words[1]}, - "host-memory fill/copy did not update the clean cached mirror"); - - gpu.SendCommandSync([&] { - Require("GpuCommandLane", "GPU memory invalidation", - resources.InvalidateMemory(fault_base, sizeof(uint32_t)), - "GPU memory invalidation did not find its mapped range"); - }); - - auto& buffer_cache = resources.GetBufferCache(); - Require("GpuCommandLane", "clean tracked fault setup", - buffer_cache.IsRegionRegistered(immediate_dst, sizeof(uint32_t)) && - !buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)), - "clean write-fault test address is not backed by a clean cached Buffer"); - constexpr uint64_t dirty_fault_address = fault_base + 0xd000; - constexpr uint32_t dirty_fault_value = 0x5a17c0deu; - constexpr uint32_t dirty_fault_stale = 0x0ddba11u; - std::memcpy(reinterpret_cast(dirty_fault_address), &dirty_fault_stale, - sizeof(dirty_fault_stale)); - gpu.SendCommandSync([&] { - auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), dirty_fault_address, - sizeof(dirty_fault_value), true, false); - Require("GpuCommandLane", "dirty tracked fault setup", dirty.owner != nullptr, - "dirty write-fault test could not create a cached Buffer"); - scheduler.Current().RetainResourceUntilFence(dirty.owner); - buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), - dirty_fault_value); - }); - Require("GpuCommandLane", "dirty tracked fault ownership", - buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)), - "dirty write-fault test address did not acquire GPU ownership"); - - std::binary_semaphore block_entered {0}; - std::binary_semaphore block_release {0}; - std::binary_semaphore clean_fault_complete {0}; - bool clean_fault_handled = false; - gpu.SendCommand([&] { - block_entered.release(); - block_release.acquire(); - }); - block_entered.acquire(); - std::jthread clean_fault_thread([&] { - clean_fault_handled = resources.HandleFault(PageFaultAccess::Write, immediate_dst); - clean_fault_complete.release(); - }); - const bool clean_fault_was_direct = - clean_fault_complete.try_acquire_for(std::chrono::seconds(1)); - if (!clean_fault_was_direct) { - block_release.release(); - clean_fault_complete.acquire(); - } - clean_fault_thread.join(); - Require("GpuCommandLane", "clean fault direct path", - clean_fault_was_direct && clean_fault_handled, - "a clean CPU write fault waited for the blocked GPU command lane"); - - std::binary_semaphore dirty_fault_complete {0}; - bool dirty_fault_handled = false; - std::jthread dirty_fault_thread([&] { - dirty_fault_handled = - resources.HandleFault(PageFaultAccess::Write, dirty_fault_address); - dirty_fault_complete.release(); - }); - const bool dirty_fault_bypassed_lane = - dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200)); - block_release.release(); - if (!dirty_fault_bypassed_lane) { - dirty_fault_complete.acquire(); - } - dirty_fault_thread.join(); - uint32_t dirty_fault_backing = 0; - std::memcpy(&dirty_fault_backing, reinterpret_cast(dirty_fault_address), - sizeof(dirty_fault_backing)); - Require("GpuCommandLane", "dirty fault synchronization", - !dirty_fault_bypassed_lane && dirty_fault_handled && - dirty_fault_backing == dirty_fault_value && - !buffer_cache.HasGpuDirtyBytes(dirty_fault_address, sizeof(dirty_fault_value)), - "a GPU-dirty write fault bypassed synchronization or lost the published bytes"); - - constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000; - constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu; - gpu.SendCommandSync([&] { - auto dirty = buffer_cache.ObtainBuffer(scheduler.Current(), empty_copy_fault_address, - sizeof(empty_copy_fault_value), true, false); - Require("GpuCommandLane", "empty-copy write setup", dirty.owner != nullptr, - "empty-copy write test could not create a cached Buffer"); - scheduler.Current().RetainResourceUntilFence(dirty.owner); - buffer_cache.FillBuffer(empty_copy_fault_address, sizeof(empty_copy_fault_value), - empty_copy_fault_value); - buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value)); - const bool gpu_owned = buffer_cache.IsRegionGpuModified( - empty_copy_fault_address, sizeof(empty_copy_fault_value)); - const bool cpu_owned = buffer_cache.IsRegionCpuModified( - empty_copy_fault_address, sizeof(empty_copy_fault_value)); - Require("GpuCommandLane", "empty-copy read ownership", - !gpu_owned && !cpu_owned, - "readback did not leave clean ownership for the queued write invalidation"); - buffer_cache.ReadMemory(empty_copy_fault_address, sizeof(empty_copy_fault_value), true); - }); - uint32_t empty_copy_fault_backing = 0; - std::memcpy(&empty_copy_fault_backing, reinterpret_cast(empty_copy_fault_address), - sizeof(empty_copy_fault_backing)); - const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified( - empty_copy_fault_address, sizeof(empty_copy_fault_value)); - const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified( - empty_copy_fault_address, sizeof(empty_copy_fault_value)); - const bool empty_copy_dirty = - buffer_cache.HasGpuDirtyBytes(empty_copy_fault_address, sizeof(empty_copy_fault_value)); - Require("GpuCommandLane", "empty-copy write ownership", - empty_copy_fault_backing == empty_copy_fault_value && - empty_copy_cpu_owned && !empty_copy_gpu_owned && !empty_copy_dirty, - "write invalidation without a remaining copy did not establish CPU ownership"); - - resources.UnmapMemory(fault_base, fault_size); - Require("GpuCommandLane", "processor fault unmap", - Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0, - "processor-fault direct-memory mapping release failed"); - Require("GpuCommandLane", "processor fault free", - Libs::LibKernel::Memory::KernelReleaseDirectMemory(fault_direct_offset, - fault_size) == 0, - "processor-fault direct-memory allocation release failed"); - - std::binary_semaphore command_entered {0}; - std::binary_semaphore release_command {0}; - std::atomic shutdown_complete {false}; - std::atomic submission_lane_entered {false}; - gpu.SendCommand([&] { - command_entered.release(); - release_command.acquire(); - gpu.Done(); - submission_lane_entered = true; - }); - command_entered.acquire(); - std::jthread shutdown_thread([&] { - gpu.Shutdown(); - shutdown_complete = true; - }); - while (!gpu.IsStopping()) { - std::this_thread::yield(); - } - Require("GpuCommandLane", "owned shutdown", !shutdown_complete.load(), - "GPU owner returned before an in-flight command completed"); - release_command.release(); - shutdown_thread.join(); - Require("GpuCommandLane", "owned shutdown completion", - shutdown_complete.load() && submission_lane_entered.load(), - "GPU owner did not drain a command that entered the submission lane"); - context.ShutdownGpu(); - std::printf("[host] %-32s ok\n", "GpuCommandLane"); - } - - void CheckUnifiedImageViewCache() { - EnsureRuntimeContext(); - constexpr const char* name = "UnifiedImageViewCache"; - CommandScheduler scheduler(Renderer(), m_runtime_context); - - ImageInfo color_info {}; - color_info.pixel_format = vk::Format::eR8G8B8A8Unorm; - color_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; - color_info.type = Prospero::ImageType::kColor2D; - color_info.extent = {8, 8, 1}; - color_info.resources = {2, 2}; - color_info.pitch = 8; - color_info.bytes_per_block = 4; - color_info.samples = 1; - color_info.tile_mode = Prospero::TileMode::kLinear; - color_info.mip_layout[0] = {0, 512, 8, 8}; - color_info.mip_layout[1] = {512, 128, 4, 4}; - - Libs::Graphics::Image color(m_runtime_context, scheduler, color_info); - const vk::ComponentMapping identity {}; - const vk::ComponentMapping bgra {vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eG, - vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eA}; - - ImageViewInfo sampled {}; - sampled.format = color_info.pixel_format; - sampled.type = vk::ImageViewType::e2D; - sampled.aspect = vk::ImageAspectFlagBits::eColor; - sampled.base_level = 0; - sampled.level_count = 1; - sampled.base_layer = 0; - sampled.layer_count = 1; - sampled.mapping = identity; - sampled.usage = vk::ImageUsageFlagBits::eSampled; - - const auto first = color.FindView(sampled); - const auto first_again = color.FindView(sampled); - auto swizzled_info = sampled; - swizzled_info.mapping = bgra; - const auto swizzled = color.FindView(swizzled_info); - auto mip_info = sampled; - mip_info.base_level = 1; - const auto mip = color.FindView(mip_info); - auto layer_info = sampled; - layer_info.base_layer = 1; - const auto layer = color.FindView(layer_info); - auto array_info = sampled; - array_info.type = vk::ImageViewType::e2DArray; - array_info.layer_count = 2; - const auto array = color.FindView(array_info); - auto reinterpreted_info = sampled; - reinterpreted_info.format = vk::Format::eR8G8B8A8Uint; - const auto reinterpreted = color.FindView(reinterpreted_info); - auto storage_info = sampled; - storage_info.usage = vk::ImageUsageFlagBits::eStorage; - const auto storage = color.FindView(storage_info); - auto attachment_info = sampled; - attachment_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - const auto attachment = color.FindView(attachment_info); - - Require(name, "color views", - first != nullptr && first_again == first && swizzled != nullptr && - swizzled != first && mip != nullptr && mip != first && layer != nullptr && - layer != first && array != nullptr && array != first && - reinterpreted != nullptr && reinterpreted != first && storage != nullptr && - storage != first && attachment == first && color.views.views.size() == 7, - "dynamic view identity omitted mapping, format, mip, layer, type, " - "or storage usage"); - - ImageInfo depth_info {}; - depth_info.pixel_format = vk::Format::eD32Sfloat; - depth_info.guest_format = Prospero::BufferFormat::k32Float; - depth_info.type = Prospero::ImageType::kColor2D; - depth_info.extent = {8, 8, 1}; - depth_info.resources = {1, 2}; - depth_info.pitch = 8; - depth_info.bytes_per_block = 4; - depth_info.samples = 1; - depth_info.tile_mode = Prospero::TileMode::kDepth; - depth_info.mip_layout[0] = {0, 512, 8, 8}; - - Libs::Graphics::Image depth(m_runtime_context, scheduler, depth_info); - ImageViewInfo depth_sampled {}; - depth_sampled.format = depth_info.pixel_format; - depth_sampled.type = vk::ImageViewType::e2D; - depth_sampled.aspect = vk::ImageAspectFlagBits::eDepth; - depth_sampled.base_level = 0; - depth_sampled.level_count = 1; - depth_sampled.base_layer = 0; - depth_sampled.layer_count = 1; - depth_sampled.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR, - vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR}; - depth_sampled.usage = vk::ImageUsageFlagBits::eSampled; - const auto depth_first = depth.FindView(depth_sampled); - auto depth_alias_info = depth_sampled; - depth_alias_info.format = vk::Format::eR32Uint; - const auto depth_alias = depth.FindView(depth_alias_info); - const auto depth_again = depth.FindView(depth_sampled); - auto depth_array_info = depth_sampled; - depth_array_info.type = vk::ImageViewType::e2DArray; - depth_array_info.layer_count = 2; - const auto depth_array = depth.FindView(depth_array_info); - auto depth_attachment_info = depth_array_info; - depth_attachment_info.mapping = {}; - depth_attachment_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto depth_attachment = depth.FindView(depth_attachment_info); - Require(name, "depth views", - depth_first != nullptr && depth_again == depth_first && - depth_alias == depth_first && depth_array != nullptr && - depth_array != depth_first && depth_attachment != nullptr && - depth_attachment != depth_array && depth.views.views.size() == 3, - "unified depth view cache lost sampled/attachment identity"); - Require( - name, "role-free backing", - color.backing.image_type == vk::ImageType::e2D && color.backing.layers == 2 && - depth.backing.image_type == vk::ImageType::e2D && - ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm, - vk::Format::eR8G8B8A8Uint) && - !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat, vk::Format::eR32Sfloat), - "role-free image backing or host format compatibility diverged"); - - auto line_info = color_info; - line_info.type = Prospero::ImageType::kColor1D; - line_info.extent = {8, 1, 1}; - line_info.resources = {2, 2}; - line_info.mip_layout[0] = {0, 64, 8, 1}; - line_info.mip_layout[1] = {64, 16, 4, 1}; - Libs::Graphics::Image line(m_runtime_context, scheduler, line_info); - auto line_view_info = sampled; - line_view_info.type = vk::ImageViewType::e1D; - const auto line_view = line.FindView(line_view_info); - auto line_array_info = line_view_info; - line_array_info.type = vk::ImageViewType::e1DArray; - line_array_info.layer_count = 2; - const auto line_array = line.FindView(line_array_info); - Require(name, "1D views", - line_view != nullptr && line_array != nullptr && line_array != line_view && - line.backing.image_type == vk::ImageType::e1D && line.backing.layers == 2, - "1D and 1D-array views did not use a first-class 1D backing"); - - auto volume_info = color_info; - volume_info.type = Prospero::ImageType::kColor3D; - volume_info.extent = {8, 8, 4}; - volume_info.resources = {2, 1}; - Libs::Graphics::Image volume(m_runtime_context, scheduler, volume_info); - auto slice_info = sampled; - slice_info.type = vk::ImageViewType::e2D; - slice_info.base_level = 1; - slice_info.base_layer = 1; - const auto slice = volume.FindView(slice_info); - auto slice_array_info = slice_info; - slice_array_info.type = vk::ImageViewType::e2DArray; - slice_array_info.base_layer = 0; - slice_array_info.layer_count = 2; - const auto slice_array = volume.FindView(slice_array_info); - Require(name, "3D slice views", - slice != nullptr && slice_array != nullptr && slice_array != slice && - static_cast(volume.backing.flags & - vk::ImageCreateFlagBits::e2DArrayCompatible), - "2D slice views of a compatible 3D backing were rejected"); - std::printf("[host] %-32s ok\n", name); - } - - void CheckBufferCacheDirtyGarbageCollection() { - constexpr const char* name = "BufferCacheDirtyGarbageCollection"; - constexpr uintptr_t base = 0x0000000200700000ull; - constexpr uint64_t allocation_size = 0x2400000; - constexpr uint64_t allocation_alignment = 0x10000; - constexpr uint64_t first_offset = 0x100; - constexpr uint64_t second_offset = 0x200; - constexpr uint64_t clean_offset = 0x300; - constexpr uint64_t unmap_offset = 0x4100; - constexpr uint64_t partial_unmap_offset = 0x8100; - constexpr uint64_t partial_unmap_survivor_offset = 0xc100; - constexpr uint64_t ring_fault_first_offset = 0x2100; - constexpr uint64_t ring_fault_second_offset = 0x2110; - constexpr uint32_t first_value = 0x10203040u; - constexpr uint32_t second_value = 0x50607080u; - constexpr uint32_t first_stale = 0x0badf00du; - constexpr uint32_t second_stale = 0xdeadbeefu; - constexpr uint32_t clean_value = 0xaabbccddu; - constexpr uint32_t unmap_value = 0x91a2b3c4u; - constexpr uint32_t partial_unmap_value = 0x62738495u; - constexpr uint32_t partial_unmap_survivor_value = 0xa6b7c8d9u; - constexpr uint32_t ring_fault_first_value = 0x0a1b2c3du; - constexpr uint32_t ring_fault_second_value = 0x4e5f6071u; - - EnsureRuntimeContext(); - auto& context = Renderer(); - CommandScheduler scheduler(context, m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - context.InitializeGpu(nullptr); - auto& gpu = context.GetGpu(); - - int64_t direct_offset = -1; - Require(name, "direct allocation", - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, - allocation_alignment, 0, &direct_offset) == 0, - "dirty-GC direct-memory allocation failed"); - void* mapped = reinterpret_cast(base); - Require(name, "direct mapping", - Libs::LibKernel::Memory::KernelMapDirectMemory(&mapped, allocation_size, 0x3, 0x10, - direct_offset, - allocation_alignment) == 0 && - mapped == reinterpret_cast(base), - "dirty-GC fixed direct-memory mapping failed"); - auto* memory = static_cast(mapped); - std::memcpy(memory + first_offset, &first_stale, sizeof(first_stale)); - std::memcpy(memory + second_offset, &second_stale, sizeof(second_stale)); - std::memcpy(memory + clean_offset, &clean_value, sizeof(clean_value)); - - { - GpuResourceManager resources(m_runtime_context, scheduler); - resources.SetGpu(&gpu); - auto& cache = resources.GetBufferCache(); - resources.MapMemory(base, allocation_size); - - const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) { - auto allocation = - cache.ObtainBuffer(scheduler.Current(), address, size, true, false); - Require(name, "dirty allocation", allocation.owner != nullptr, - "dirty-GC buffer allocation failed"); - scheduler.Current().RetainResourceUntilFence(allocation.owner); - }; - - constexpr uint64_t index_offset = 0x180000; - constexpr uint64_t index_page = BufferCache::CACHING_PAGE_SIZE; - constexpr uint64_t index_span = 3 * index_page; - const uint64_t index_begin = base + index_offset; - Require(name, "registered-range empty lookup", - !cache.IsRegionRegistered(index_begin, index_span), - "empty Buffer cache reported a registered range"); - auto index_left = cache.ObtainBuffer(scheduler.Current(), index_begin + 0x100, - sizeof(uint32_t), false, false); - auto index_right = cache.ObtainBuffer(scheduler.Current(), - index_begin + 2 * index_page + 0x100, - sizeof(uint32_t), false, false); - Require(name, "registered-range owners", - index_left.owner != nullptr && index_right.owner != nullptr, - "failed to create disjoint Buffer index owners"); - scheduler.Current().RetainResourceUntilFence(index_left.owner); - scheduler.Current().RetainResourceUntilFence(index_right.owner); - Require( - name, "registered-range boundaries", - cache.IsRegionRegistered(index_begin, 1) && - cache.IsRegionRegistered(index_begin + index_page - 1, 1) && - !cache.IsRegionRegistered(index_begin - 1, 1) && - !cache.IsRegionRegistered(index_begin + index_page, index_page) && - cache.IsRegionRegistered(index_begin + index_page, index_page + 1) && - !cache.IsRegionRegistered(index_begin + index_span, 1) && - cache.IsRegionRegistered(index_begin - 1, index_span + 2), - "indexed lookup mishandled a half-open boundary, gap, or broad overlap"); - auto index_bridge = cache.ObtainBuffer(scheduler.Current(), index_begin + index_page - 1, - index_page + 2, false, false); - Require(name, "registered-range merge", - index_bridge.owner != nullptr && - cache.IsRegionRegistered(index_begin + index_page, index_page), - "bridging acquisition did not publish its merged Buffer range"); - scheduler.Current().RetainResourceUntilFence(index_bridge.owner); - cache.UnmapMemory(index_begin, index_span); - Require(name, "registered-range removal", - !cache.IsRegionRegistered(index_begin, index_span), - "unmapped Buffer owner remained in the registered-range index"); - - MarkGpuWrite(base + first_offset, sizeof(first_value)); - MarkGpuWrite(base + second_offset, sizeof(second_value)); - cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); - cache.FillBuffer(base + second_offset, sizeof(second_value), second_value); - auto& download = BufferCacheTestAccess::DownloadBuffer(cache); - auto* fixed_download = &download; - const auto fixed_download_handle = download.Handle(); - auto [ring_tail, ring_tail_offset] = - download.Map(download.Size() - sizeof(uint32_t), 4); - Require(name, "fault-ring tail reservation", - ring_tail != nullptr && ring_tail_offset == 0, - "failed to position the reusable download ring near wrap"); - download.Commit(); - MarkGpuWrite(base + ring_fault_first_offset, sizeof(ring_fault_first_value)); - MarkGpuWrite(base + ring_fault_second_offset, sizeof(ring_fault_second_value)); - cache.FillBuffer(base + ring_fault_first_offset, sizeof(ring_fault_first_value), - ring_fault_first_value); - cache.FillBuffer(base + ring_fault_second_offset, sizeof(ring_fault_second_value), - ring_fault_second_value); - Require(name, "fault-ring wrapped batch", - resources.HandleFault(PageFaultAccess::Read, base + ring_fault_first_offset), - "fault readback could not wrap a live download-ring tick"); - uint64_t expected_packing_offset = 128; - uint64_t expected_packing_alignment = 1; - if (!download.IsCoherent()) { - Require(name, "fault-ring atom policy", - StreamBufferTestAccess::NormalizeReservation( - false, - m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize, - expected_packing_offset, expected_packing_alignment), - "failed to calculate non-coherent fault-ring stride"); - } - auto [packing_probe, packing_offset] = download.Map(1, 1, false); - Require(name, "adjacent download reservation stride", - packing_probe != nullptr && packing_offset == expected_packing_offset, - "adjacent fault downloads did not reserve an atom-safe stride"); - download.Commit(); - uint32_t ring_fault_first_backing = 0; - uint32_t ring_fault_second_backing = 0; - std::memcpy(&ring_fault_first_backing, memory + ring_fault_first_offset, - sizeof(ring_fault_first_backing)); - std::memcpy(&ring_fault_second_backing, memory + ring_fault_second_offset, - sizeof(ring_fault_second_backing)); - Require(name, "fault-ring wrapped contents", - ring_fault_first_backing == ring_fault_first_value && - ring_fault_second_backing == ring_fault_second_value && - &BufferCacheTestAccess::DownloadBuffer(cache) == fixed_download && - download.Handle() == fixed_download_handle && - download.Size() == (32ull << 20), - "wrapped fault batch published incorrect disjoint ranges"); - - for (uint32_t tick = 0; tick < 160; tick++) { - cache.RunGarbageCollector(); - } - Require(name, "age before pressure", cache.IsRegionRegistered(base, allocation_size), - "buffer was reclaimed without memory pressure"); - BufferCacheTestAccess::SetGarbageCollectionThresholds( - cache, 0, std::numeric_limits::max()); - const auto gc_submission_tick = scheduler.CurrentTick(); - cache.RunGarbageCollector(); - Require(name, "dirty retirement", !cache.IsRegionRegistered(base, allocation_size), - "aged GPU-dirty buffer survived pressured " - "collection"); - - uint32_t first_before_completion = 0; - uint32_t second_before_completion = 0; - Libs::LibKernel::Memory::TryReadBacking(base + first_offset, &first_before_completion, - sizeof(first_before_completion)); - Libs::LibKernel::Memory::TryReadBacking(base + second_offset, &second_before_completion, - sizeof(second_before_completion)); - Require(name, "deferred dirty retirement", - first_before_completion == first_stale && - second_before_completion == second_stale && - cache.IsRegionGpuModified(base + first_offset, sizeof(first_value)) && - cache.IsRegionGpuModified(base + second_offset, sizeof(second_value)) && - cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) && - cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)) && - scheduler.CurrentTick() == gc_submission_tick, - "buffer GC published bytes or cleared dirty " - "ownership before GPU completion"); - scheduler.FinishCurrent(); - - uint32_t first_backing = 0; - uint32_t second_backing = 0; - uint32_t clean_backing = 0; - std::memcpy(&first_backing, memory + first_offset, sizeof(first_backing)); - std::memcpy(&second_backing, memory + second_offset, sizeof(second_backing)); - std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing)); - Require(name, "downloaded contents", - first_backing == first_value && second_backing == second_value && - clean_backing == clean_value && - !cache.IsRegionGpuModified(base + first_offset, sizeof(first_value)) && - !cache.IsRegionGpuModified(base + second_offset, sizeof(second_value)) && - !cache.HasGpuDirtyBytes(base + first_offset, sizeof(first_value)) && - !cache.HasGpuDirtyBytes(base + second_offset, sizeof(second_value)), - "dirty GC did not publish exact ranges before " - "clearing broad page ownership"); - - auto unmap_allocation = cache.ObtainBuffer(scheduler.Current(), base + unmap_offset, - sizeof(unmap_value), true, false); - Require(name, "direct-unmap allocation", unmap_allocation.owner != nullptr, - "direct-unmap buffer allocation failed"); - scheduler.Current().RetainResourceUntilFence(unmap_allocation.owner); - cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value); - cache.UnmapMemory(base + 0x4000, 0x4000); - uint32_t unmap_backing = 0; - std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing)); - Require(name, "direct-unmap contents", - unmap_backing == unmap_value && - !cache.IsRegionRegistered(base + 0x4000, 0x4000), - "direct dirty unmap cleared tracking before " - "publishing bytes"); - - auto partial_unmap_allocation = - cache.ObtainBuffer(scheduler.Current(), base + 0x8000, 0x8000, true, false); - Require(name, "partial-unmap allocation", partial_unmap_allocation.owner != nullptr, - "cross-page cached buffer allocation failed"); - scheduler.Current().RetainResourceUntilFence(partial_unmap_allocation.owner); - cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), - partial_unmap_value); - cache.FillBuffer(base + partial_unmap_survivor_offset, - sizeof(partial_unmap_survivor_value), partial_unmap_survivor_value); - cache.UnmapMemory(base + 0x8000, 0x4000); - auto survivor = - cache.ObtainBuffer(scheduler.Current(), base + partial_unmap_survivor_offset, - sizeof(partial_unmap_survivor_value), false, true); - Require(name, "partial-unmap survivor", survivor.buffer != nullptr, - "still-mapped cached-buffer remainder could not be recreated"); - if (survivor.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(survivor.owner); - } - auto partial_unmap_readback = - CreateHostBuffer(name, sizeof(partial_unmap_survivor_value), - vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy survivor_copy {survivor.offset, 0, - sizeof(partial_unmap_survivor_value)}; - scheduler.Current().Handle().copyBuffer(survivor.buffer, partial_unmap_readback.buffer, - 1, &survivor_copy); - vk::BufferMemoryBarrier survivor_barrier {}; - survivor_barrier.sType = vk::StructureType::eBufferMemoryBarrier; - survivor_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - survivor_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - survivor_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - survivor_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - survivor_barrier.buffer = partial_unmap_readback.buffer; - survivor_barrier.size = partial_unmap_readback.size; - scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eHost, {}, 0, - nullptr, 1, &survivor_barrier, 0, nullptr); - scheduler.Finish(); - Require(name, "partial-unmap survivor contents", - ReadBuffer(name, partial_unmap_readback, 1) == - std::vector {partial_unmap_survivor_value}, - "partial unmap recreated its still-mapped remainder from " - "uninitialized native bytes"); - DestroyBuffer(&partial_unmap_readback); - - constexpr uint64_t large_offset = 0x10000; - constexpr uint64_t large_size = 31ull * 1024 * 1024; - constexpr uint32_t large_value = 0x5aa55aa5u; - constexpr uint32_t large_stale = 0x12345678u; - std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale)); - std::memcpy(memory + large_offset + large_size - sizeof(large_stale), &large_stale, - sizeof(large_stale)); - auto large_allocation = cache.ObtainBuffer(scheduler.Current(), base + large_offset, - large_size, true, false); - Require(name, "near-capacity dirty allocation", large_allocation.owner != nullptr, - "failed to allocate the near-capacity dirty native buffer"); - scheduler.Current().RetainResourceUntilFence(large_allocation.owner); - cache.FillBuffer(base + large_offset, large_size, large_value); - for (uint32_t tick = 0; tick <= 160; tick++) { - cache.RunGarbageCollector(); - } - Require(name, "near-capacity deferred retirement", - !cache.IsRegionRegistered(base + large_offset, large_size), - "near-capacity dirty Buffer survived pressured collection"); - uint32_t large_before_completion = 0; - Libs::LibKernel::Memory::TryReadBacking(base + large_offset, &large_before_completion, - sizeof(large_before_completion)); - Require(name, "near-capacity backing remained deferred", - large_before_completion == large_stale, - "near-capacity Buffer GC published before its scheduler tick"); - scheduler.FinishCurrent(); - const auto large_image_source = - cache.ObtainBufferForImage(base + large_offset, sizeof(large_value)); - Require(name, "fixed download after Buffer retirement", - large_image_source.buffer != nullptr && - &BufferCacheTestAccess::DownloadBuffer(cache) == fixed_download && - fixed_download->Handle() == fixed_download_handle && - fixed_download->Size() == (32ull << 20), - "image acquisition replaced the shared Buffer download stream"); - uint32_t large_first = 0; - uint32_t large_last = 0; - Libs::LibKernel::Memory::TryReadBacking(base + large_offset, &large_first, - sizeof(large_first)); - Libs::LibKernel::Memory::TryReadBacking(base + large_offset + large_size - - sizeof(large_last), - &large_last, sizeof(large_last)); - Require(name, "near-capacity Buffer publication contents", - large_first == large_value && large_last == large_value, - "near-capacity Buffer GC did not publish its complete transfer"); - - constexpr uint64_t grouped_first_offset = 0x10000; - constexpr uint64_t grouped_second_offset = 0x1200000; - constexpr uint64_t grouped_owner_size = 17ull * 1024 * 1024; - constexpr uint32_t grouped_first_value = 0x1122aabbu; - constexpr uint32_t grouped_second_value = 0x3344ccddu; - constexpr uint32_t grouped_stale = 0; - static_assert(grouped_owner_size < (32ull << 20) && - grouped_owner_size * 2 > (32ull << 20)); - Libs::LibKernel::Memory::WriteBacking(base + grouped_first_offset, &grouped_stale, - sizeof(grouped_stale)); - Libs::LibKernel::Memory::WriteBacking(base + grouped_second_offset, &grouped_stale, - sizeof(grouped_stale)); - auto grouped_first = cache.ObtainBuffer( - scheduler.Current(), base + grouped_first_offset, grouped_owner_size, true, false); - auto grouped_second = cache.ObtainBuffer( - scheduler.Current(), base + grouped_second_offset, grouped_owner_size, true, false); - Require(name, "per-owner GC allocation", - grouped_first.owner != nullptr && grouped_second.owner != nullptr && - grouped_first.owner != grouped_second.owner, - "disjoint GC candidates merged into one Buffer owner"); - scheduler.Current().RetainResourceUntilFence(grouped_first.owner); - scheduler.Current().RetainResourceUntilFence(grouped_second.owner); - cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, grouped_first_value); - cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, - grouped_second_value); - for (uint32_t tick = 0; tick <= 160; tick++) { - cache.RunGarbageCollector(); - } - Require(name, "per-owner fixed-ring retirement", - !cache.IsRegionRegistered(base + grouped_first_offset, grouped_owner_size) && - !cache.IsRegionRegistered(base + grouped_second_offset, grouped_owner_size) && - &BufferCacheTestAccess::DownloadBuffer(cache) == fixed_download && - fixed_download->Handle() == fixed_download_handle, - "GC aggregated disjoint retirees or replaced the download ring"); - scheduler.FinishCurrent(); - uint32_t grouped_first_backing = 0; - uint32_t grouped_second_backing = 0; - Libs::LibKernel::Memory::TryReadBacking( - base + grouped_first_offset, &grouped_first_backing, sizeof(grouped_first_backing)); - Libs::LibKernel::Memory::TryReadBacking(base + grouped_second_offset, - &grouped_second_backing, - sizeof(grouped_second_backing)); - Require(name, "per-owner fixed-ring contents", - grouped_first_backing == grouped_first_value && - grouped_second_backing == grouped_second_value, - "per-owner GC transfers lost data while wrapping the fixed ring"); - - constexpr uint64_t disjoint_owner_offset = 0x2140000; - constexpr uint64_t disjoint_owner_size = 0x8000; - constexpr uint64_t disjoint_dirty_offset = disjoint_owner_offset + 0x100; - constexpr uint64_t disjoint_new_owner_offset = disjoint_owner_offset + 0x5000; - constexpr uint32_t disjoint_value = 0x89abcdefu; - constexpr uint32_t disjoint_stale = 0x76543210u; - std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale, sizeof(disjoint_stale)); - auto wide_owner = cache.ObtainBuffer(scheduler.Current(), base + disjoint_owner_offset, - disjoint_owner_size, false, true); - Require(name, "disjoint retirement owner", wide_owner.owner != nullptr, - "failed to create the clean cross-page retirement owner"); - scheduler.Current().RetainResourceUntilFence(wide_owner.owner); - auto dirty_alias = cache.ObtainBuffer(scheduler.Current(), base + disjoint_dirty_offset, - sizeof(disjoint_value), true, false); - if (dirty_alias.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(dirty_alias.owner); - } - cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), disjoint_value); - for (uint32_t tick = 0; tick <= 160; tick++) { - cache.RunGarbageCollector(); - } - Require(name, "disjoint deferred retirement", - !cache.IsRegionRegistered(base + disjoint_owner_offset, disjoint_owner_size), - "cross-page owner survived pressured collection"); - uint32_t disjoint_before_unmap = 0; - Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, - &disjoint_before_unmap, - sizeof(disjoint_before_unmap)); - Require(name, "disjoint retirement remained deferred", - disjoint_before_unmap == disjoint_stale, - "whole-owner Buffer publication completed before synchronization"); - scheduler.FinishCurrent(); - cache.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000); - uint32_t disjoint_after_unmap = 0; - Libs::LibKernel::Memory::TryReadBacking( - base + disjoint_dirty_offset, &disjoint_after_unmap, sizeof(disjoint_after_unmap)); - Require( - name, "disjoint synchronized unmap", - disjoint_after_unmap == disjoint_value && - !cache.HasGpuDirtyBytes(base + disjoint_dirty_offset, sizeof(disjoint_value)), - "disjoint unmap lost the completed whole-owner publication"); - auto disjoint_new_owner = - cache.ObtainBuffer(scheduler.Current(), base + disjoint_new_owner_offset, - sizeof(disjoint_value), true, false); - Require(name, "disjoint post-publication reacquire", - disjoint_new_owner.owner != nullptr, - "disjoint acquisition failed after whole-owner publication"); - scheduler.Current().RetainResourceUntilFence(disjoint_new_owner.owner); - uint32_t disjoint_backing = 0; - Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, &disjoint_backing, - sizeof(disjoint_backing)); - Require(name, "disjoint retirement contents", disjoint_backing == disjoint_value, - "old retirement callback lost dirty bytes or retained ownership " - "after a disjoint reacquire"); - cache.ReadMemory(base + disjoint_new_owner_offset, sizeof(disjoint_value)); - - constexpr uint64_t reacquire_owner_offset = 0x2200000; - constexpr uint64_t reacquire_owner_size = 0x8000; - constexpr uint64_t reacquire_dirty_offset = reacquire_owner_offset + 0x100; - constexpr uint64_t reacquire_disjoint_offset = reacquire_owner_offset + 0x5000; - constexpr uint32_t reacquire_value = 0xc001d00du; - constexpr uint32_t reacquire_stale = 0x0badf00du; - std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale, sizeof(reacquire_stale)); - auto reacquire_owner = - cache.ObtainBuffer(scheduler.Current(), base + reacquire_owner_offset, - reacquire_owner_size, false, true); - Require(name, "reacquire retirement owner", reacquire_owner.owner != nullptr, - "failed to create a fresh cross-page retirement owner"); - scheduler.Current().RetainResourceUntilFence(reacquire_owner.owner); - auto reacquire_dirty = - cache.ObtainBuffer(scheduler.Current(), base + reacquire_dirty_offset, - sizeof(reacquire_value), true, false); - if (reacquire_dirty.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(reacquire_dirty.owner); - } - cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), - reacquire_value); - for (uint32_t tick = 0; tick <= 160; tick++) { - cache.RunGarbageCollector(); - } - Require(name, "reacquire deferred retirement", - !cache.IsRegionRegistered(base + reacquire_owner_offset, reacquire_owner_size), - "fresh cross-page owner survived pressured collection"); - uint32_t reacquire_before = 0; - Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, - &reacquire_before, sizeof(reacquire_before)); - Require(name, "reacquire publication remained deferred", - reacquire_before == reacquire_stale, - "fresh whole-owner publication completed before reacquisition"); - scheduler.FinishCurrent(); - auto reacquired = - cache.ObtainBuffer(scheduler.Current(), base + reacquire_disjoint_offset, - sizeof(reacquire_value), true, false); - Require(name, "independent disjoint post-publication reacquire", - reacquired.owner != nullptr, - "clean disjoint-half acquisition failed after publication"); - scheduler.Current().RetainResourceUntilFence(reacquired.owner); - uint32_t reacquire_after = 0; - Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, &reacquire_after, - sizeof(reacquire_after)); - Require( - name, "independent disjoint reacquire contents", - reacquire_after == reacquire_value && - !cache.HasGpuDirtyBytes(base + reacquire_dirty_offset, sizeof(reacquire_value)), - "disjoint-half acquisition failed to publish the retired " - "owner's dirty prefix"); - cache.ReadMemory(base + reacquire_disjoint_offset, sizeof(reacquire_value)); - - resources.SetGpu(nullptr); - resources.UnmapMemory(base, allocation_size); - scheduler.Finish(); - } - context.ShutdownGpu(); - Require(name, "unmap direct backing", - Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, - "dirty-GC direct-memory mapping release failed"); - Require( - name, "release direct backing", - Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0, - "dirty-GC direct-memory allocation release failed"); - std::printf("[host] %-32s ok\n", name); - } - - void CheckUnifiedTextureCacheFlow() { - constexpr const char* name = "UnifiedTextureCacheFlow"; - constexpr uintptr_t base = 0x0000000200600000ull; - constexpr uint64_t allocation_size = 0x2800000; - constexpr uint64_t allocation_alignment = 0x200000; - EnsureRuntimeContext(); - auto& context = Renderer(); - CommandScheduler scheduler(context, m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - context.InitializeGpu(nullptr); - auto& gpu = context.GetGpu(); - int64_t direct_offset = -1; - Require(name, "direct allocation", - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, - allocation_alignment, 0, &direct_offset) == 0, - "cache test direct-memory allocation failed"); - void* mapped = reinterpret_cast(base); - Require(name, "direct mapping", - Libs::LibKernel::Memory::KernelMapDirectMemory(&mapped, allocation_size, 0x3, 0x10, - direct_offset, - allocation_alignment) == 0 && - mapped == reinterpret_cast(base), - "fixed cache direct-memory mapping failed"); - auto* memory = static_cast(mapped); - const uint32_t initial = 0x44332211u; - std::memcpy(memory, &initial, sizeof(initial)); - - { - GpuResourceManager resources(m_runtime_context, scheduler); - resources.SetGpu(&gpu); - auto& texture_cache = resources.GetTextureCache(); - const auto [narrow_download, narrow_download_offset] = - TextureCacheTestAccess::MapDownload(texture_cache, 4, 4); - const auto [wide_download, wide_download_offset] = - TextureCacheTestAccess::MapDownload(texture_cache, 1, 16); - Require(name, "image download alignment", - narrow_download != nullptr && narrow_download_offset % 4 == 0 && - wide_download != nullptr && wide_download_offset % 16 == 0, - "wide/block image readback was not aligned to its texel block"); - resources.MapMemory(base, allocation_size); - - ImageDesc sampled {}; - sampled.type = BindingType::Texture; - sampled.info.data = {base, sizeof(initial)}; - sampled.info.pixel_format = vk::Format::eR8G8B8A8Srgb; - sampled.info.guest_format = - Prospero::BufferFormat::k8_8_8_8Srgb; - sampled.info.type = Prospero::ImageType::kColor2D; - sampled.info.extent = {1, 1, 1}; - sampled.info.resources = {1, 1}; - sampled.info.pitch = 1; - sampled.info.bytes_per_block = 4; - sampled.info.samples = 1; - sampled.info.tile_mode = Prospero::TileMode::kLinear; - sampled.info.mip_layout[0] = {0, 4, 1, 1}; - sampled.view_info.format = sampled.info.pixel_format; - sampled.view_info.type = vk::ImageViewType::e2D; - sampled.view_info.aspect = vk::ImageAspectFlagBits::eColor; - sampled.view_info.usage = vk::ImageUsageFlagBits::eSampled; - - auto& command = scheduler.Current(); - auto first_desc = sampled; - const auto first = texture_cache.FindImage(first_desc); - auto repeated_desc = sampled; - const auto repeated = texture_cache.FindImage(repeated_desc); - auto compatible_desc = sampled; - compatible_desc.info.pixel_format = vk::Format::eR8G8B8A8Uint; - compatible_desc.info.guest_format = - Prospero::BufferFormat::k8_8_8_8UInt; - compatible_desc.view_info.format = compatible_desc.info.pixel_format; - const auto compatible = texture_cache.FindImage(compatible_desc); - Require(name, "normalized FindImage", - first && repeated == first && compatible == first && - texture_cache.GetImage(first).info.pixel_format == - vk::Format::eR8G8B8A8Srgb, - "registered compatible backing did not reuse one ImageId"); - - const auto IsOnlyImage = [](const std::vector& ids, ImageId expected) { - return ids.size() == 1 && ids.front() == expected; - }; - const auto exact_byte_miss = - TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false); - const auto touched_page_hit = - TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true); - const auto next_page_miss = - TextureCacheTestAccess::FindImages(texture_cache, base + 0x1000, 1, true); - Require(name, "exact and 4-KiB page filtering", - exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) && - next_page_miss.empty(), - "coarse candidates did not preserve exact-byte and touched-page semantics"); - - const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) { - ImageInfo info {}; - info.data = {address, size}; - info.extent = {1, 1, 1}; - info.resources = {1, 1}; - info.samples = 1; - return info; - }; - const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000); - const auto spanning = TextureCacheTestAccess::InsertImage(texture_cache, spanning_info); - const auto spanning_results = TextureCacheTestAccess::FindImages( - texture_cache, spanning_info.data.address, spanning_info.data.size, false); - Require(name, "one-MiB cross-page deduplication", - spanning && IsOnlyImage(spanning_results, spanning) && - TextureCacheTestAccess::PageOwnerCount(texture_cache, - spanning_info.data.address) == 1 && - TextureCacheTestAccess::PageOwnerCount( - texture_cache, spanning_info.data.End() - 1) == 1, - "an image spanning two coarse pages was missing or returned more than once"); - TextureCacheTestAccess::DeleteImage(texture_cache, spanning); - Require(name, "cross-page owner cleanup", - TextureCacheTestAccess::PageOwnerCount(texture_cache, - spanning_info.data.address) == 0 && - TextureCacheTestAccess::PageOwnerCount( - texture_cache, spanning_info.data.End() - 1) == 0 && - TextureCacheTestAccess::FindImages(texture_cache, - spanning_info.data.address, - spanning_info.data.size, false) - .empty(), - "cross-page unregister left a stale coarse-page membership"); - - const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100); - const auto shared_first = - TextureCacheTestAccess::InsertImage(texture_cache, shared_info); - const auto shared_second = - TextureCacheTestAccess::InsertImage(texture_cache, shared_info); - const auto shared_results = TextureCacheTestAccess::FindImages( - texture_cache, shared_info.data.address, shared_info.data.size, false); - Require(name, "shared coarse-page registration", - shared_results.size() == 2 && - std::find(shared_results.begin(), shared_results.end(), shared_first) != - shared_results.end() && - std::find(shared_results.begin(), shared_results.end(), shared_second) != - shared_results.end(), - "two registered owners were not retained in one coarse page"); - TextureCacheTestAccess::DeleteImage(texture_cache, shared_first); - const auto shared_survivor = TextureCacheTestAccess::FindImages( - texture_cache, shared_info.data.address, shared_info.data.size, false); - Require(name, "shared coarse-page unregister", - IsOnlyImage(shared_survivor, shared_second) && - TextureCacheTestAccess::PageOwnerCount(texture_cache, - shared_info.data.address) == 1, - "unregistering one owner removed its coarse-page neighbor"); - TextureCacheTestAccess::DeleteImage(texture_cache, shared_second); - Require(name, "final shared coarse-page unregister", - TextureCacheTestAccess::PageOwnerCount(texture_cache, - shared_info.data.address) == 0, - "the final shared owner remained registered"); - - constexpr uint64_t large_owner_size = 64ull * 1024 * 1024; - const auto large_info = MakeOwnershipInfo(base + 0x800000, large_owner_size); - const auto large_owner = - TextureCacheTestAccess::InsertImage(texture_cache, large_info); - Require(name, "production one-MiB registration granularity", - TextureCacheTestAccess::OwnedPageCount( - texture_cache, large_info.data.address, large_info.data.size, large_owner) == 64, - "64 MiB image registration did not create exactly 64 coarse memberships"); - TextureCacheTestAccess::DeleteImage(texture_cache, large_owner); - Require(name, "production one-MiB unregister granularity", - TextureCacheTestAccess::OwnedPageCount( - texture_cache, large_info.data.address, large_info.data.size, large_owner) == 0, - "large image unregister left coarse memberships behind"); - - const ImageId stale {first.index, first.generation + 1}; - TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale); - const auto stale_filtered = - TextureCacheTestAccess::FindImages(texture_cache, base, sizeof(initial), false); - Require(name, "stale page owner filtering", - IsOnlyImage(stale_filtered, first) && - TextureCacheTestAccess::RemovePageOwner(texture_cache, base, stale), - "a stale generation escaped the direct page-owner lookup"); - - TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX); - const auto wrap_results = - TextureCacheTestAccess::FindImages(texture_cache, base, sizeof(initial), false); - Require(name, "page-owner query epoch wrap", - IsOnlyImage(wrap_results, first) && - TextureCacheTestAccess::QueryEpoch(texture_cache) == 1, - "query deduplication failed while wrapping its epoch"); - - auto& image = texture_cache.GetImage(first); - constexpr uint32_t final_sampled_value = 0x88776655u; - Require(name, "sampled write between discovery and acquisition", - resources.HandleFault(PageFaultAccess::Write, sampled.info.data.address), - "sampled image did not accept a CPU update after discovery"); - std::memcpy(memory, &final_sampled_value, sizeof(final_sampled_value)); - Require(name, "sampled final-acquisition precondition", image.IsCpuDirty(), - "the between-phase CPU write did not dirty the discovered image"); - const auto first_view = texture_cache.FindTexture(first, first_desc); - const auto repeated_view = texture_cache.FindTexture(repeated, repeated_desc); - auto mapped_view_info = sampled.view_info; - mapped_view_info.mapping.r = vk::ComponentSwizzle::eB; - mapped_view_info.mapping.b = vk::ComponentSwizzle::eR; - const auto mapped_view = image.FindView(mapped_view_info); - Require(name, "dynamic views", - first_view != nullptr && repeated_view == first_view && - mapped_view != nullptr && mapped_view != first_view && - image.views.views.size() == 2 && !image.usage.texture && - !image.IsGpuModified() && !image.IsCpuDirty() && - !TextureCacheTestAccess::PendingDownload(texture_cache, first), - "TextureCache::FindTexture did not preserve sampled view " - "identity, final refresh, and cache-only ownership"); - - ImageInfo chain = sampled.info; - chain.data = {0x10000, 0x8000}; - chain.extent = {8, 8, 1}; - chain.resources = {2, 4}; - chain.pitch = 8; - chain.mip_layout[0] = {0, 0x4000, 8, 8}; - chain.mip_layout[1] = {0x4000, 0x4000, 4, 4}; - ImageInfo subresource = chain; - subresource.data = {0x16000, 0x1000}; - subresource.extent = {4, 4, 1}; - subresource.resources = {1, 1}; - subresource.pitch = 4; - subresource.mip_layout[0] = {0, 0x1000, 4, 4}; - const auto mip = subresource.MipOf(chain); - Require(name, "overlap resolution", - mip == 1 && subresource.SliceOf(chain, mip) == 2 && - ImageRangeOverlaps(chain.data, subresource.data) && - ImagePageRangesOverlap(chain.data, subresource.data), - "normalized mip/slice overlap did not resolve"); - - auto exact_desc = compatible_desc; - const auto exact = texture_cache.FindImage(exact_desc, true); - Require( - name, "exact-format coexistence", - exact && exact != first && TextureCacheTestAccess::Contains(texture_cache, first) && - texture_cache.GetImage(first).info.pixel_format == sampled.info.pixel_format && - texture_cache.GetImage(exact).info.pixel_format == - compatible_desc.info.pixel_format, - "exact-format lookup replaced or synchronized its compatible " - "cache record"); - auto compatible_after_exact_desc = compatible_desc; - const auto compatible_after_exact = - texture_cache.FindImage(compatible_after_exact_desc); - Require(name, "last compatible format winner", compatible_after_exact == exact, - "non-exact lookup did not retain the last compatible " - "registration"); - - auto null_desc = sampled; - null_desc.info.data = {}; - auto null_repeat_desc = null_desc; - const auto null_image = texture_cache.FindImage(null_desc); - const auto null_repeat = texture_cache.FindImage(null_repeat_desc); - Require(name, "null image", - null_image && null_repeat == null_image && null_image != exact, - "typed null-image lookup was not stable"); - - auto MakeLinearDesc = [&](uint64_t address, uint64_t size, vk::Format format, - Prospero::BufferFormat guest_format, Prospero::ImageType type, - vk::Extent3D extent, uint32_t layers, - uint32_t bytes_per_block, uint32_t samples) { - ImageDesc desc {}; - desc.type = BindingType::Texture; - desc.info.data = {address, size}; - desc.info.pixel_format = format; - desc.info.guest_format = guest_format; - desc.info.type = type; - desc.info.extent = extent; - desc.info.resources = {1, layers}; - desc.info.pitch = extent.width; - desc.info.bytes_per_block = bytes_per_block; - desc.info.samples = samples; - desc.info.tile_mode = Prospero::TileMode::kLinear; - desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; - desc.view_info.format = format; - if (type == Prospero::ImageType::kColor3D) { - desc.view_info.type = vk::ImageViewType::e3D; - } else if (layers > 1) { - desc.view_info.type = vk::ImageViewType::e2DArray; - } else { - desc.view_info.type = vk::ImageViewType::e2D; - } - desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; - desc.view_info.layer_count = layers; - desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; - return desc; - }; - const auto HostReadBarrier = [&](vk::Buffer buffer, uint64_t size, - vk::PipelineStageFlags source_stage, - vk::AccessFlags source_access) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = source_access; - barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer; - barrier.offset = 0; - barrier.size = size; - scheduler.Current().Handle().pipelineBarrier(source_stage, - vk::PipelineStageFlagBits::eHost, {}, - 0, nullptr, 1, &barrier, 0, nullptr); - }; - const auto TransferReadBarrier = [&](vk::Buffer buffer, uint64_t size) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer; - barrier.offset = 0; - barrier.size = size; - scheduler.Current().Handle().pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer, {}, - 0, nullptr, 1, &barrier, 0, nullptr); - }; - - // A formatted Buffer read must use the private + VulkanHarness() { Init(); } + ~VulkanHarness() { Destroy(); } + + VulkanHarness(const VulkanHarness &) = delete; + VulkanHarness &operator=(const VulkanHarness &) = delete; + + struct Buffer { + vk::Buffer buffer = nullptr; + vk::DeviceMemory memory = nullptr; + vk::DeviceSize size = 0; + bool coherent = false; + }; + + struct Image { + vk::Image image = nullptr; + vk::DeviceMemory memory = nullptr; + vk::ImageView view = nullptr; + vk::Format format = vk::Format::eUndefined; + vk::ImageLayout layout = vk::ImageLayout::eUndefined; + u32 width = 0; + u32 height = 0; + u32 layers = 1; + u32 mip_levels = 1; + u32 dwords_per_pixel = 0; + }; + + [[nodiscard]] vk::Device Device() const { return m_device; } + [[nodiscard]] GraphicContext &RuntimeContext() { + EnsureRuntimeContext(); + return m_runtime_context; + } + [[nodiscard]] RenderContext &RuntimeRenderer() { + EnsureRuntimeContext(); + return Renderer(); + } + + void CheckCommandPoolGrowth() { + EnsureRuntimeContext(); + std::vector> commands; + for (uint32_t i = 0; i < 12; i++) { + commands.push_back( + std::make_unique(Renderer().GetCommandScheduler())); + } + for (auto &command : commands) { + Require("CommandPoolGrowth", "allocation", !command->IsInvalid(), + "unified command pool failed to grow"); + command->Begin(); + command->End(); + command->Execute(); + } + for (auto &command : commands) { + command->WaitForFence(); + } + std::printf("[host] %-32s ok\n", "CommandPoolGrowth"); + } + + void CheckSchedulerTimeline() { + EnsureRuntimeContext(); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + + const auto first_tick = scheduler.CurrentTick(); + uint32_t completed = 0; + scheduler.DeferOperation([owned = std::make_unique(1), + &completed] { completed = *owned; }); + scheduler.Flush(); + scheduler.Wait(first_tick); + Require("SchedulerTimeline", "first tick", + scheduler.CurrentTick() == first_tick + 1 && + scheduler.IsFree(first_tick) && completed == 1, + "timeline wait did not release its deferred operation"); + + const auto second_tick = scheduler.CurrentTick(); + std::atomic priority_in_callback{false}; + scheduler.DeferPriorityOperation([owned = std::make_unique(2), + &completed, &priority_in_callback] { + priority_in_callback = CommandScheduler::InDeferredOperation(); + completed = *owned; + }); + scheduler.Finish(); + scheduler.DrainPriorityOperations(); + Require("SchedulerTimeline", "finish", + scheduler.CurrentTick() == second_tick + 1 && + scheduler.IsFree(second_tick) && completed == 2 && + priority_in_callback.load() && + !CommandScheduler::InDeferredOperation(), + "scheduler completion drain lost its tick or callback context"); + + const auto implicit_flush_tick = scheduler.CurrentTick(); + scheduler.DeferOperation([owned = std::make_unique(3), + &completed] { completed = *owned; }); + scheduler.Wait(implicit_flush_tick); + Require("SchedulerTimeline", "implicit flush", + scheduler.CurrentTick() == implicit_flush_tick + 1 && + scheduler.IsFree(implicit_flush_tick) && completed == 3, + "waiting for the current tick did not flush it"); + + vk::SemaphoreTypeCreateInfo timeline_type{}; + timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo; + timeline_type.semaphoreType = vk::SemaphoreType::eTimeline; + vk::SemaphoreCreateInfo timeline_create{}; + timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo; + timeline_create.pNext = &timeline_type; + vk::Semaphore external_timeline = nullptr; + Require("SchedulerTimeline", "external timeline create", + m_runtime_context.device.createSemaphore(&timeline_create, nullptr, + &external_timeline) == + vk::Result::eSuccess && + external_timeline != nullptr, + "failed to create the external timeline semaphore"); + + constexpr uint64_t external_wait_value = 9; + const auto external_wait_tick = scheduler.CurrentTick(); + SubmitInfo external_wait; + external_wait.AddWait(external_timeline, external_wait_value); + constexpr size_t blocked_submission_count = 6; + std::array blocked_handles{}; + for (size_t i = 0; i < blocked_submission_count; ++i) { + blocked_handles[i] = scheduler.Current().Handle(); + if (i == 0) { + scheduler.Flush(external_wait); + } else { + scheduler.Flush(); + } + } + bool distinct_handles = true; + for (size_t i = 0; i < blocked_handles.size(); ++i) { + for (size_t j = i + 1; j < blocked_handles.size(); ++j) { + distinct_handles &= blocked_handles[i] != blocked_handles[j]; + } + } + const auto last_blocked_tick = scheduler.CurrentTick() - 1; + Require("SchedulerTimeline", "external timeline overflow", + distinct_handles && + last_blocked_tick == + external_wait_tick + blocked_submission_count - 1 && + !scheduler.IsFree(last_blocked_tick), + "blocked submissions did not grow the command-buffer pool"); + vk::SemaphoreSignalInfo signal_info{}; + signal_info.sType = vk::StructureType::eSemaphoreSignalInfo; + signal_info.semaphore = external_timeline; + signal_info.value = external_wait_value; + Require("SchedulerTimeline", "external timeline signal", + m_runtime_context.device.signalSemaphore(&signal_info) == + vk::Result::eSuccess, + "failed to signal the external timeline semaphore"); + scheduler.Wait(last_blocked_tick); + m_runtime_context.device.destroySemaphore(external_timeline, nullptr); + scheduler.Flush(); + scheduler.Flush(); + Require( + "SchedulerTimeline", "timeline pool reuse", + std::ranges::find(blocked_handles, scheduler.Current().Handle()) != + blocked_handles.end(), + "completed command buffers were not reused after timeline progress"); + scheduler.Shutdown(); + + CommandScheduler draining(Renderer(), m_runtime_context); + HW::Context drain_registers{}; + HW::UserConfig drain_user_config{}; + HW::Shader drain_shaders{}; + draining.Begin(drain_registers, drain_user_config, drain_shaders); + uint32_t reentrant = 0; + draining.DeferOperation([&draining, &reentrant] { + reentrant++; + draining.DeferOperation([&reentrant] { reentrant++; }); + }); + + std::binary_semaphore priority_entered{0}; + std::binary_semaphore release_priority{0}; + draining.DeferPriorityOperation( + [&draining, &priority_entered, &release_priority] { + priority_entered.release(); + draining.Shutdown(); + release_priority.acquire(); + }); + std::jthread shutdown_thread([&draining] { draining.Shutdown(); }); + priority_entered.acquire(); + std::jthread second_shutdown_thread([&draining] { draining.Shutdown(); }); + + std::atomic concurrent_completed{false}; + std::jthread concurrent_defer([&draining, &concurrent_completed] { + draining.DeferOperation( + [&concurrent_completed] { concurrent_completed = true; }); + }); + release_priority.release(); + shutdown_thread.join(); + second_shutdown_thread.join(); + concurrent_defer.join(); + Require("SchedulerTimeline", "shutdown drain", + reentrant == 2 && concurrent_completed.load(), + "shutdown lost reentrant or concurrent deferred work"); + std::printf("[host] %-32s ok\n", "SchedulerTimeline"); + } + + void CheckGpuMappedRangeLifecycle() { + EnsureRuntimeContext(); + auto &context = Renderer(); + CommandScheduler scheduler(context, m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + context.InitializeGpu(nullptr); + auto &gpu = context.GetGpu(); + GpuResourceManager resources(m_runtime_context, scheduler); + resources.SetGpu(&gpu); + + constexpr uint64_t base = 0x0000000200000000ull; + constexpr uint64_t page = 0x4000; + resources.MapMemory(base, page * 4); + resources.MapMemory(base + page * 2, page * 4); + Require("GpuMappedRangeLifecycle", "union", + resources.IsMapped(base, page * 6) && + !resources.IsMapped(base, page * 7), + "overlapping maps did not form one interval union"); + + resources.UnmapMemory(base + page * 2, page * 2); + Require("GpuMappedRangeLifecycle", "subtract", + resources.IsMapped(base, page * 2) && + resources.IsMapped(base + page * 4, page * 2) && + !resources.IsMapped(base, page * 6), + "partial unmap did not punch the expected interval hole"); + + resources.UnmapMemory(base + page * 2, page * 2); + Require("GpuMappedRangeLifecycle", "idempotent unmap", + resources.IsMapped(base, page * 2) && + resources.IsMapped(base + page * 4, page * 2), + "unmapping an absent interval changed neighboring mappings"); + + resources.UnmapMemory(base, page * 6); + Require("GpuMappedRangeLifecycle", "clear", + !resources.IsMapped(base, page * 6), + "full unmap did not clear the interval union"); + + constexpr uint64_t old_prt = base + page * 8; + constexpr uint64_t new_prt = base + page * 16; + resources.MapMemory(old_prt, page * 4); + resources.UnmapMemory(old_prt, page * 4); + resources.MapMemory(new_prt, page * 6); + Require("GpuMappedRangeLifecycle", "PRT replacement", + !resources.IsMapped(old_prt, page * 4) && + resources.IsMapped(new_prt, page * 6), + "old-unmap/new-map did not replace full PRT coverage"); + + resources.SetGpu(nullptr); + scheduler.Finish(); + context.ShutdownGpu(); + std::printf("[host] %-32s ok\n", "GpuMappedRangeLifecycle"); + } + + void CheckStreamBufferRing() { + EnsureRuntimeContext(); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + GpuResourceManager resources(m_runtime_context, scheduler); + auto &cache = resources.GetBufferCache(); + constexpr std::array, 4> utilities{{ + {MemoryUsage::Upload, 512ull << 20}, + {MemoryUsage::Stream, 64ull << 20}, + {MemoryUsage::Download, 32ull << 20}, + {MemoryUsage::DeviceLocal, 128ull << 20}, + }}; + std::array handles{}; + for (size_t i = 0; i < utilities.size(); i++) { + auto &utility = cache.GetUtilityBuffer(utilities[i].first); + handles[i] = utility.Handle(); + Require("StreamBufferRing", "utility ownership", + utility.Usage() == utilities[i].first && + utility.Size() == utilities[i].second && + utility.Handle() != nullptr, + "BufferCache utility buffer does not match the fixed layout"); + } + for (size_t i = 0; i < handles.size(); i++) { + for (size_t j = i + 1; j < handles.size(); j++) { + Require("StreamBufferRing", "utility identity", + handles[i] != handles[j], + "utility usages alias the same native buffer"); + } + } + auto *fixed_download = &cache.GetUtilityBuffer(MemoryUsage::Download); + const auto fixed_handle = fixed_download->Handle(); + const auto [oversized_download, oversized_offset] = + fixed_download->Map(fixed_download->Size() + 4, 64, false); + const auto [download_probe, download_probe_offset] = + fixed_download->Map(64, 64, false); + Require("StreamBufferRing", "fixed download utility", + oversized_download == nullptr && oversized_offset == 0 && + download_probe != nullptr && download_probe_offset == 0 && + &cache.GetUtilityBuffer(MemoryUsage::Download) == + fixed_download && + fixed_download->Size() == (32ull << 20) && + fixed_download->Handle() == fixed_handle, + "oversized download replaced or corrupted the fixed shared ring"); + fixed_download->Commit(); + + std::vector full_stream(64ull << 20, 0x5a); + const auto utility_tick = scheduler.CurrentTick(); + auto full_binding = + cache.UploadTransient(full_stream.data(), full_stream.size(), 16); + const std::array overflow_data{ + 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, + 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5, 0xa5}; + auto overflow_binding = + cache.UploadTransient(overflow_data.data(), overflow_data.size(), 16); + auto overflow_owner = std::static_pointer_cast( + overflow_binding.owner); + Require( + "StreamBufferRing", "current tick overflow", + full_binding.owner == nullptr && + full_binding.buffer == + cache.GetUtilityBuffer(MemoryUsage::Stream).Handle() && + overflow_owner != nullptr && + overflow_binding.buffer == overflow_owner->Handle() && + overflow_binding.buffer != full_binding.buffer && + overflow_owner->Mapped().size() == overflow_data.size() && + std::memcmp(overflow_owner->Mapped().data(), overflow_data.data(), + overflow_data.size()) == 0 && + scheduler.CurrentTick() == utility_tick, + "current-tick stream overflow submitted partial state or lost bytes"); + + const auto atom = + m_runtime_context.physical_device_properties.limits.nonCoherentAtomSize; + uint64_t policy_size = 17; + uint64_t policy_alignment = 12; + uint64_t coherent_size = policy_size; + uint64_t coherent_alignment = policy_alignment; + Require("StreamBufferRing", "non-coherent reservation policy", + StreamBufferTestAccess::NormalizeReservation( + false, 256, policy_size, policy_alignment) && + policy_size == 256 && policy_alignment == 768 && + StreamBufferTestAccess::NormalizeReservation( + true, 256, coherent_size, coherent_alignment) && + coherent_size == 17 && coherent_alignment == 12, + "non-coherent reservations are not isolated at the atom/texel LCM"); + const auto small_ring_size = std::max(64, atom * 2); + StreamBuffer stream(m_runtime_context, scheduler, MemoryUsage::Upload, + small_ring_size); + const auto first_size = + stream.IsCoherent() ? small_ring_size - 16 : small_ring_size - atom + 1; + constexpr uint64_t blocked_size = 24; + const auto [first, first_offset] = stream.Map(first_size, 16, false); + Require("StreamBufferRing", "first allocation", + first != nullptr && first_offset == 0, + "initial stream allocation failed"); + std::memset(first, 0x5a, static_cast(first_size)); + stream.Commit(); + + const auto [oversize, oversize_offset] = + stream.Map(small_ring_size + 1, 1, false); + Require("StreamBufferRing", "oversize", + oversize == nullptr && oversize_offset == 0, + "oversize allocation was accepted"); + const auto [blocked, blocked_offset] = stream.Map(blocked_size, 1, false); + const auto [blocked_again, blocked_again_offset] = + stream.Map(blocked_size, 1, false); + Require("StreamBufferRing", "transactional failure", + blocked == nullptr && blocked_offset == 0 && + blocked_again == nullptr && blocked_again_offset == 0, + "failed wrap mutated stream state"); + + scheduler.Flush(); + const auto [wrapped, wrapped_offset] = stream.Map(blocked_size, 1); + Require("StreamBufferRing", "watched wrap", + wrapped != nullptr && wrapped_offset == 0, + "completed watch did not release the wrapped allocation"); + stream.Commit(); + + const auto download_alignment = atom / std::gcd(atom, 12) * 12; + const auto download_ring_size = std::max( + 64, download_alignment + std::max(atom, 16)); + StreamBuffer download(m_runtime_context, scheduler, MemoryUsage::Download, + download_ring_size); + const auto [download_data, download_offset] = download.Map(16, 8); + Require("StreamBufferRing", "download allocation", + download_data != nullptr && download_offset == 0, + "download stream is not host visible"); + download.Commit(); + scheduler.Flush(); + const auto [second_download, second_download_offset] = download.Map(16, 12); + Require("StreamBufferRing", "download atom isolation", + second_download != nullptr && + (download.IsCoherent() || + (download_offset % atom == 0 && + second_download_offset % atom == 0 && + second_download_offset >= download_offset + atom)) && + second_download_offset % 12 == 0, + "download reservations share a non-coherent atom or lost caller " + "alignment"); + download.Commit(); + + scheduler.Finish(); + download.Invalidate(download_offset, 16); + std::printf("[host] %-32s ok\n", "StreamBufferRing"); + } + + void CheckGpuCommandLane() { + EnsureRuntimeContext(); + auto &context = Renderer(); + context.InitializeGpu(nullptr); + auto &gpu = context.GetGpu(); + + const auto caller_thread = std::this_thread::get_id(); + std::thread::id gpu_thread; + std::thread::id nested_thread; + uint32_t order = 0; + gpu.SendCommand([&order] { order = 1; }); + gpu.SendCommandSync([&] { + gpu_thread = std::this_thread::get_id(); + Require("GpuCommandLane", "FIFO", order == 1, + "synchronous command overtook an older host command"); + Require("GpuCommandLane", "GPU context", Gpu::IsGpuThread(), + "host command did not run on the GPU thread"); + gpu.SendCommandSync( + [&nested_thread] { nested_thread = std::this_thread::get_id(); }); + order = 2; + }); + Require( + "GpuCommandLane", "dispatch thread", + order == 2 && gpu_thread != caller_thread && + nested_thread == gpu_thread, + "host command did not run on the GPU thread or nested sync deadlocked"); + + { + const auto make_release_mem = + [](uint32_t data_sel, uint32_t interrupt_selector, void *destination, + uint64_t value, uint32_t action = 0x28u, + uint32_t gcr_cntl = 1u << 9u) { + std::array packet{}; + const auto address = reinterpret_cast(destination); + const auto event_index = (action >= 0x2fu ? 6u : 5u); + packet[0] = KYTY_PM4(8, Pm4::IT_NOP, Pm4::R_RELEASE_MEM); + packet[1] = action | (event_index << 8u) | (gcr_cntl << 12u); + packet[2] = (interrupt_selector << 24u) | (data_sel << 29u); + packet[3] = static_cast(address); + packet[4] = static_cast(address >> 32u); + packet[5] = static_cast(value); + packet[6] = static_cast(value >> 32u); + packet[7] = 0x123u; + return packet; + }; + + vk::SemaphoreTypeCreateInfo timeline_type{}; + timeline_type.sType = vk::StructureType::eSemaphoreTypeCreateInfo; + timeline_type.semaphoreType = vk::SemaphoreType::eTimeline; + vk::SemaphoreCreateInfo timeline_create{}; + timeline_create.sType = vk::StructureType::eSemaphoreCreateInfo; + timeline_create.pNext = &timeline_type; + vk::Semaphore blocked_timeline = nullptr; + Require("GpuCommandLane", "blocked timeline create", + m_runtime_context.device.createSemaphore( + &timeline_create, nullptr, &blocked_timeline) == + vk::Result::eSuccess && + blocked_timeline != nullptr, + "failed to create the blocked GPU timeline"); + + constexpr uint64_t release_tick = 1; + auto &gpu_scheduler = context.GetCommandScheduler(); + auto processor = std::make_unique(context); + gpu.SendCommandSync([&] { + processor->BufferInit(); + SubmitInfo blocked_submit; + blocked_submit.AddWait(blocked_timeline, release_tick); + gpu_scheduler.Flush(blocked_submit); + }); + + std::binary_semaphore parser_complete{0}; + alignas(uint64_t) uint64_t interrupt_only_gds_label = UINT64_MAX; + alignas(uint64_t) uint64_t cb_db_release_label = UINT64_MAX; + bool parser_kept_nonblocking_boundaries = false; + std::jthread no_gpu_wait([&] { + gpu.SendCommandSync([&] { + processor->BufferInit(); + const auto tick_before = gpu_scheduler.CurrentTick(); + processor->TriggerEvent(0x16u, 0u); + auto packet = make_release_mem(0, 0, nullptr, 0); + Pm4Execution execution; + const auto result = + processor->Process(execution, packet.data(), packet.size()); + processor->IncrementDe(); + const bool cache_and_counter_did_not_submit = + result == Pm4ProcessResult::Complete && + gpu_scheduler.CurrentTick() == tick_before; + + const auto cb_db_tick = gpu_scheduler.CurrentTick(); + auto cb_db_release = + make_release_mem(2, 0, &cb_db_release_label, 0, 0x14u, 0); + Pm4Execution cb_db_execution; + const auto cb_db_result = processor->Process( + cb_db_execution, cb_db_release.data(), cb_db_release.size()); + const bool cb_db_release_did_not_submit = + cb_db_result == Pm4ProcessResult::Complete && + cb_db_release_label == 0 && + gpu_scheduler.CurrentTick() == cb_db_tick; + + auto gds_interrupt_only = + make_release_mem(5, 1, &interrupt_only_gds_label, 1ull << 16u); + Pm4Execution gds_interrupt_execution; + const auto interrupt_tick = gpu_scheduler.CurrentTick(); + const auto interrupt_result = processor->Process( + gds_interrupt_execution, gds_interrupt_only.data(), + gds_interrupt_only.size()); + parser_kept_nonblocking_boundaries = + cache_and_counter_did_not_submit && + cb_db_release_did_not_submit && + interrupt_result == Pm4ProcessResult::Complete && + gpu_scheduler.CurrentTick() == interrupt_tick + 1 && + interrupt_only_gds_label == UINT64_MAX; + }); + parser_complete.release(); + }); + const bool parser_did_not_wait_gpu = + parser_complete.try_acquire_for(std::chrono::seconds(2)); + + vk::SemaphoreSignalInfo signal_info{}; + signal_info.sType = vk::StructureType::eSemaphoreSignalInfo; + signal_info.semaphore = blocked_timeline; + signal_info.value = release_tick; + Require("GpuCommandLane", "blocked timeline signal", + m_runtime_context.device.signalSemaphore(&signal_info) == + vk::Result::eSuccess, + "failed to release the blocked GPU timeline"); + no_gpu_wait.join(); + gpu.SendCommandSync([&] { gpu_scheduler.Finish(); }); + m_runtime_context.device.destroySemaphore(blocked_timeline, nullptr); + Require("GpuCommandLane", "nonblocking GPU packets", + parser_did_not_wait_gpu && parser_kept_nonblocking_boundaries, + "a cache event, GL2 writeback, DE counter, or interrupt boundary " + "used the wrong " + "submission behavior"); + + alignas(uint64_t) uint64_t release_label = 0; + alignas(uint64_t) uint64_t gds_label = UINT64_MAX; + bool release_mem_submission_counts = false; + gpu.SendCommandSync([&] { + processor->BufferInit(); + + auto immediate = make_release_mem(1, 0, &release_label, 0x11223344u); + Pm4Execution immediate_execution; + const auto immediate_tick = gpu_scheduler.CurrentTick(); + const auto immediate_result = processor->Process( + immediate_execution, immediate.data(), immediate.size()); + const bool immediate_split_once = + gpu_scheduler.CurrentTick() == immediate_tick + 1; + + auto gds = make_release_mem(5, 0, &gds_label, 1ull << 16u); + Pm4Execution gds_execution; + const auto gds_tick = gpu_scheduler.CurrentTick(); + const auto gds_result = + processor->Process(gds_execution, gds.data(), gds.size()); + const bool gds_waited_once = + gpu_scheduler.CurrentTick() == gds_tick + 1; + + auto interrupt_only = make_release_mem(0, 4, nullptr, 0); + Pm4Execution interrupt_execution; + const auto interrupt_tick = gpu_scheduler.CurrentTick(); + const auto interrupt_result = processor->Process( + interrupt_execution, interrupt_only.data(), interrupt_only.size()); + const bool interrupt_split_once = + gpu_scheduler.CurrentTick() == interrupt_tick + 1; + + auto gds_interrupt = make_release_mem(5, 2, &gds_label, 1ull << 16u); + Pm4Execution gds_interrupt_execution; + const auto gds_interrupt_tick = gpu_scheduler.CurrentTick(); + const auto gds_interrupt_result = + processor->Process(gds_interrupt_execution, gds_interrupt.data(), + gds_interrupt.size()); + const bool gds_interrupt_waited_once = + gpu_scheduler.CurrentTick() == gds_interrupt_tick + 1; + + release_mem_submission_counts = + immediate_result == Pm4ProcessResult::Complete && + immediate_split_once && gds_result == Pm4ProcessResult::Complete && + gds_waited_once && interrupt_result == Pm4ProcessResult::Complete && + interrupt_split_once && + gds_interrupt_result == Pm4ProcessResult::Complete && + gds_interrupt_waited_once; + }); + gpu.SendCommandSync([&] { gpu_scheduler.Finish(); }); + Require("GpuCommandLane", "RELEASE_MEM submission counts", + release_mem_submission_counts && + static_cast(release_label) == 0x11223344u && + static_cast(gds_label) == 0, + "RELEASE_MEM lost its required split/readback or retained a " + "redundant GPU wait"); + } + + alignas(uint32_t) uint32_t packet_marker_a = 0; + alignas(uint32_t) uint32_t packet_marker_b = 0; + const auto write_packet = [](uint32_t *packet, uint32_t *destination, + uint32_t value) { + const auto destination_address = reinterpret_cast(destination); + packet[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + packet[1] = 0; + packet[2] = static_cast(destination_address); + packet[3] = static_cast(destination_address >> 32u); + packet[4] = value; + }; + std::array polling_loop{}; + for (size_t i = 0; i < polling_loop.size() - 4; i += 2) { + polling_loop[i] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); + polling_loop[i + 1] = 0; + } + const auto loop_address = reinterpret_cast(polling_loop.data()); + polling_loop[1020] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + polling_loop[1021] = static_cast(loop_address); + polling_loop[1022] = static_cast(loop_address >> 32u); + polling_loop[1023] = + 0x0f200000u | static_cast(polling_loop.size()); + + std::array polling_commands{}; + write_packet(polling_commands.data(), &packet_marker_a, 11); + polling_commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + polling_commands[6] = static_cast(loop_address); + polling_commands[7] = static_cast(loop_address >> 32u); + polling_commands[8] = + 0x0f200000u | static_cast(polling_loop.size()); + write_packet(polling_commands.data() + 9, &packet_marker_b, 22); + std::binary_semaphore stream_gate_entered{0}; + std::binary_semaphore stream_gate_release{0}; + gpu.SendCommand([&] { + stream_gate_entered.release(); + stream_gate_release.acquire(); + }); + gpu.Submit(polling_commands.data(), polling_commands.size(), nullptr, 0); + stream_gate_entered.acquire(); + stream_gate_release.release(); + uint32_t packet_marker_a_at_callback = UINT32_MAX; + uint32_t packet_marker_b_at_callback = UINT32_MAX; + for (uint32_t attempt = 0; attempt < 8 && packet_marker_a_at_callback != 11; + attempt++) { + gpu.SendCommandSync([&] { + if (packet_marker_a == 11 && packet_marker_b == 0) { + packet_marker_a_at_callback = packet_marker_a; + packet_marker_b_at_callback = packet_marker_b; + polling_loop[1020] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); + polling_loop[1021] = 0; + polling_loop[1022] = KYTY_PM4(2, Pm4::IT_NOP, Pm4::R_ZERO); + polling_loop[1023] = 0; + } + }); + std::this_thread::yield(); + } + gpu.Done(); + Require("GpuCommandLane", "packet-boundary command polling", + packet_marker_a_at_callback == 11 && + packet_marker_b_at_callback == 0 && packet_marker_a == 11 && + packet_marker_b == 22, + "host command waited for an entire non-suspended PM4 stream"); + + uint32_t label = 0; + uint32_t prefix = 0; + uint32_t suffix = 0; + const auto address = [](const void *value) { + return reinterpret_cast(value); + }; + std::array commands{}; + commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[1] = 0; + commands[2] = static_cast(address(&prefix)); + commands[3] = static_cast(address(&prefix) >> 32u); + commands[4] = 11; + commands[5] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); + commands[6] = static_cast(address(&label)); + commands[7] = static_cast(address(&label) >> 32u); + commands[8] = UINT32_MAX; + commands[9] = 1; + commands[10] = 0x10u | 3u; + commands[12] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[13] = 0; + commands[14] = static_cast(address(&suffix)); + commands[15] = static_cast(address(&suffix) >> 32u); + commands[16] = 22; + gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, + 0); + + std::binary_semaphore ordered_started{0}; + std::atomic ordered_finished{false}; + uint32_t ordered_suffix = 0; + std::jthread ordered([&] { + ordered_started.release(); + gpu.Done(); + ordered_suffix = suffix; + ordered_finished = true; + }); + ordered_started.acquire(); + gpu.SendCommandSync([&] { + Require("GpuCommandLane", "submit done barrier", !ordered_finished.load(), + "submit done returned before a queued submission"); + label = 1; + }); + ordered.join(); + Require("GpuCommandLane", "ordered completion", + prefix == 11 && suffix == 22 && ordered_suffix == 22 && + ordered_finished.load(), + "submit done did not drain prior PM4 work"); + + auto &resources = context.GetGpuResources(); + constexpr uint64_t empty_unmap_base = 0x0000000200400000ull; + constexpr uint64_t empty_unmap_size = 0x4000; + resources.MapMemory(empty_unmap_base, empty_unmap_size); + label = 0; + prefix = 0; + suffix = 0; + gpu.Submit(commands.data(), static_cast(commands.size()), nullptr, + 0); + + std::binary_semaphore unmap_complete{0}; + std::jthread unmap_thread([&] { + resources.UnmapMemory(empty_unmap_base, empty_unmap_size); + unmap_complete.release(); + }); + const bool unmap_returned = + unmap_complete.try_acquire_for(std::chrono::seconds(2)); + gpu.SendCommandSync([&] { label = 1; }); + if (!unmap_returned) { + unmap_complete.acquire(); + } + unmap_thread.join(); + gpu.Done(); + Require("GpuCommandLane", "unmap queue progress", + unmap_returned && + !resources.IsMapped(empty_unmap_base, empty_unmap_size) && + prefix == 11 && suffix == 22, + "an unrelated unmap waited for a blocked PM4 submission"); + + auto &scheduler = context.GetCommandScheduler(); + std::atomic normal_completed{false}; + gpu.SendCommandSync( + [&] { scheduler.DeferOperation([&] { normal_completed = true; }); }); + resources.MapMemory(empty_unmap_base, empty_unmap_size); + resources.UnmapMemory(empty_unmap_base, empty_unmap_size); + Require("GpuCommandLane", "unmap native completion", + normal_completed.load() && + !resources.IsMapped(empty_unmap_base, empty_unmap_size), + "unmap returned before an earlier native guest-memory callback"); + + std::binary_semaphore priority_entered{0}; + std::binary_semaphore release_priority{0}; + gpu.SendCommandSync([&] { + scheduler.DeferPriorityOperation([&] { + priority_entered.release(); + release_priority.acquire(); + }); + scheduler.Flush(); + }); + priority_entered.acquire(); + resources.MapMemory(empty_unmap_base, empty_unmap_size); + + std::binary_semaphore priority_unmap_entered{0}; + std::binary_semaphore priority_unmap_complete{0}; + std::jthread priority_unmap_thread([&] { + gpu.SendCommandSync([&] { + priority_unmap_entered.release(); + resources.UnmapMemory(empty_unmap_base, empty_unmap_size); + }); + priority_unmap_complete.release(); + }); + priority_unmap_entered.acquire(); + const bool unmap_overtook_priority = + priority_unmap_complete.try_acquire_for(std::chrono::seconds(1)); + release_priority.release(); + if (!unmap_overtook_priority) { + priority_unmap_complete.acquire(); + } + priority_unmap_thread.join(); + Require("GpuCommandLane", "unmap priority ordering", + !unmap_overtook_priority && + !resources.IsMapped(empty_unmap_base, empty_unmap_size), + "unmap returned before an earlier guest-memory callback"); + + constexpr uintptr_t fault_base = 0x0000000200500000ull; + constexpr uint64_t fault_size = 0x10000; + int64_t fault_direct_offset = -1; + Require("GpuCommandLane", "processor fault direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + fault_size, fault_size, 0, &fault_direct_offset) == 0, + "processor-fault direct-memory allocation failed"); + void *fault_memory = reinterpret_cast(fault_base); + Require("GpuCommandLane", "processor fault direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &fault_memory, fault_size, 0x3, 0x10, fault_direct_offset, + fault_size) == 0, + "processor-fault fixed direct-memory mapping failed"); + Require("GpuCommandLane", "processor fault allocation", + fault_memory == reinterpret_cast(fault_base), + "fixed processor-fault allocation failed"); + resources.MapMemory(fault_base, fault_size); + + constexpr uint64_t immediate_dst = fault_base + 0x1000; + constexpr uint64_t immediate_memory_dst = fault_base + 0x2000; + constexpr uint64_t memory_src = fault_base + 0x4000; + constexpr uint64_t memory_copy_dst = fault_base + 0x6000; + constexpr uint64_t memory_dst = fault_base + 0x8000; + constexpr uint64_t l2_copy_dst = fault_base + 0xa000; + constexpr uint64_t clean_cached_fill = fault_base + 0x3000; + constexpr uint64_t clean_cached_copy = fault_base + 0x3008; + constexpr uint64_t clean_cached_source = fault_base + 0x3010; + constexpr uint64_t clean_cached_readback = fault_base + 0xb000; + constexpr std::array source_words{0x11223344u, 0x55667788u}; + constexpr std::array clean_copy_words{0x13579bdfu, + 0x2468ace0u}; + std::memcpy(reinterpret_cast(memory_src), source_words.data(), + sizeof(source_words)); + std::memcpy(reinterpret_cast(clean_cached_source), + clean_copy_words.data(), sizeof(clean_copy_words)); + + std::array dma_commands{}; + size_t dma_cursor = 0; + const auto append_dma = [&](uint8_t dst_sel, uint64_t dst, uint8_t src_sel, + uint64_t src, uint32_t bytes) { + auto *packet = dma_commands.data() + dma_cursor; + dma_cursor += 7; + packet[0] = KYTY_PM4(7, Pm4::IT_DMA_DATA, 0); + packet[1] = (static_cast(dst_sel) << 20u) | + (static_cast(src_sel) << 29u); + packet[2] = static_cast(src); + packet[3] = static_cast(src >> 32u); + packet[4] = static_cast(dst); + packet[5] = static_cast(dst >> 32u); + packet[6] = bytes; + }; + constexpr uint64_t gds_immediate_offset = 0; + constexpr uint64_t gds_memory_offset = 0x10; + constexpr uint32_t immediate_value = 0xa1b2c3d4u; + append_dma(1, gds_immediate_offset, 2, immediate_value, + sizeof(source_words)); + append_dma(0, immediate_dst, 1, gds_immediate_offset, sizeof(source_words)); + append_dma(1, gds_memory_offset, 0, memory_src, sizeof(source_words)); + append_dma(0, memory_dst, 1, gds_memory_offset, sizeof(source_words)); + append_dma(0, immediate_memory_dst, 2, immediate_value, + sizeof(source_words)); + append_dma(0, memory_copy_dst, 0, memory_src, sizeof(source_words)); + append_dma(3, l2_copy_dst, 3, memory_src, sizeof(source_words)); + Require("GpuCommandLane", "DMA_DATA packet assembly", + dma_cursor == dma_commands.size(), + "DMA_DATA GDS packet stream has the wrong size"); + gpu.Submit(dma_commands.data(), static_cast(dma_commands.size()), + nullptr, 0); + gpu.Done(); + constexpr uint32_t clean_fill_value = 0xdecafbad; + gpu.SendCommandSync([&] { + auto &buffer_cache = resources.GetBufferCache(); + Require("GpuCommandLane", "clean cached setup", + buffer_cache.IsRegionRegistered(clean_cached_fill, + sizeof(source_words)) && + buffer_cache.HasGpuDirtyBytes(immediate_dst, + sizeof(source_words)), + "DMA did not establish a cached page with a dirty sibling"); + + buffer_cache.FillBuffer(clean_cached_fill, sizeof(source_words), + clean_fill_value); + buffer_cache.CopyBuffer(clean_cached_copy, clean_cached_source, + sizeof(clean_copy_words)); + Require("GpuCommandLane", "clean cached DMA ownership", + !buffer_cache.HasGpuDirtyBytes(clean_cached_fill, + sizeof(source_words)) && + !buffer_cache.HasGpuDirtyBytes(clean_cached_copy, + sizeof(clean_copy_words)) && + buffer_cache.HasGpuDirtyBytes(immediate_dst, + sizeof(source_words)), + "clean cached DMA acquired ownership or released a byte-disjoint " + "dirty sibling"); + + constexpr uint64_t gds_clean_mirror = 0x20; + buffer_cache.CopyBuffer(gds_clean_mirror, clean_cached_fill, + 2 * sizeof(source_words), true, false); + buffer_cache.CopyBuffer(clean_cached_readback, gds_clean_mirror, + 2 * sizeof(source_words), false, true); + }); + Require("GpuCommandLane", "DMA_DATA immediate GDS readback", + resources.HandleFault(PageFaultAccess::Read, immediate_dst), + "immediate-to-GDS-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA memory GDS readback", + resources.HandleFault(PageFaultAccess::Read, memory_dst), + "memory-to-GDS-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA immediate memory readback", + resources.HandleFault(PageFaultAccess::Read, immediate_memory_dst), + "immediate-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA memory copy readback", + resources.HandleFault(PageFaultAccess::Read, memory_copy_dst), + "memory-to-memory copy did not publish GPU bytes"); + Require("GpuCommandLane", "DMA_DATA L2 readback", + resources.HandleFault(PageFaultAccess::Read, l2_copy_dst), + "MemoryUsingL2 copy did not publish GPU bytes"); + Require("GpuCommandLane", "clean cached mirror readback", + resources.HandleFault(PageFaultAccess::Read, clean_cached_readback), + "clean host DMA was not reflected in the cached buffer"); + std::array immediate_words{}; + std::array copied_words{}; + std::array immediate_memory_words{}; + std::array memory_copy_words{}; + std::array l2_copy_words{}; + std::array clean_cached_words{}; + std::memcpy(immediate_words.data(), + reinterpret_cast(immediate_dst), + sizeof(immediate_words)); + std::memcpy(copied_words.data(), reinterpret_cast(memory_dst), + sizeof(copied_words)); + std::memcpy(immediate_memory_words.data(), + reinterpret_cast(immediate_memory_dst), + sizeof(immediate_memory_words)); + std::memcpy(memory_copy_words.data(), + reinterpret_cast(memory_copy_dst), + sizeof(memory_copy_words)); + std::memcpy(l2_copy_words.data(), + reinterpret_cast(l2_copy_dst), + sizeof(l2_copy_words)); + std::memcpy(clean_cached_words.data(), + reinterpret_cast(clean_cached_readback), + sizeof(clean_cached_words)); + Require("GpuCommandLane", "DMA_DATA immediate GDS contents", + immediate_words == + std::array{immediate_value, immediate_value}, + "immediate-to-GDS-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA memory GDS contents", + copied_words == source_words, + "memory-to-GDS-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA immediate memory contents", + immediate_memory_words == + std::array{immediate_value, immediate_value}, + "immediate-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA memory copy contents", + memory_copy_words == source_words, + "memory-to-memory bytes do not match"); + Require("GpuCommandLane", "DMA_DATA L2 contents", + l2_copy_words == source_words, "MemoryUsingL2 bytes do not match"); + Require("GpuCommandLane", "clean cached DMA contents", + clean_cached_words == std::array{clean_fill_value, + clean_fill_value, + clean_copy_words[0], + clean_copy_words[1]}, + "host-memory fill/copy did not update the clean cached mirror"); + + gpu.SendCommandSync([&] { + Require("GpuCommandLane", "GPU memory invalidation", + resources.InvalidateMemory(fault_base, sizeof(uint32_t)), + "GPU memory invalidation did not find its mapped range"); + }); + + auto &buffer_cache = resources.GetBufferCache(); + Require("GpuCommandLane", "clean tracked fault setup", + buffer_cache.IsRegionRegistered(immediate_dst, sizeof(uint32_t)) && + !buffer_cache.HasGpuDirtyBytes(immediate_dst, sizeof(uint32_t)), + "clean write-fault test address is not backed by a clean cached " + "Buffer"); + constexpr uint64_t dirty_fault_address = fault_base + 0xd000; + constexpr uint32_t dirty_fault_value = 0x5a17c0deu; + constexpr uint32_t dirty_fault_stale = 0x0ddba11u; + std::memcpy(reinterpret_cast(dirty_fault_address), + &dirty_fault_stale, sizeof(dirty_fault_stale)); + gpu.SendCommandSync([&] { + auto dirty = + buffer_cache.ObtainBuffer(scheduler.Current(), dirty_fault_address, + sizeof(dirty_fault_value), true, false); + Require("GpuCommandLane", "dirty tracked fault setup", + dirty.owner != nullptr, + "dirty write-fault test could not create a cached Buffer"); + scheduler.Current().RetainResourceUntilFence(dirty.owner); + buffer_cache.FillBuffer(dirty_fault_address, sizeof(dirty_fault_value), + dirty_fault_value); + }); + Require("GpuCommandLane", "dirty tracked fault ownership", + buffer_cache.HasGpuDirtyBytes(dirty_fault_address, + sizeof(dirty_fault_value)), + "dirty write-fault test address did not acquire GPU ownership"); + + std::binary_semaphore block_entered{0}; + std::binary_semaphore block_release{0}; + std::binary_semaphore clean_fault_complete{0}; + bool clean_fault_handled = false; + gpu.SendCommand([&] { + block_entered.release(); + block_release.acquire(); + }); + block_entered.acquire(); + std::jthread clean_fault_thread([&] { + clean_fault_handled = + resources.HandleFault(PageFaultAccess::Write, immediate_dst); + clean_fault_complete.release(); + }); + const bool clean_fault_was_direct = + clean_fault_complete.try_acquire_for(std::chrono::seconds(1)); + if (!clean_fault_was_direct) { + block_release.release(); + clean_fault_complete.acquire(); + } + clean_fault_thread.join(); + Require("GpuCommandLane", "clean fault direct path", + clean_fault_was_direct && clean_fault_handled, + "a clean CPU write fault waited for the blocked GPU command lane"); + + std::binary_semaphore dirty_fault_complete{0}; + bool dirty_fault_handled = false; + std::jthread dirty_fault_thread([&] { + dirty_fault_handled = + resources.HandleFault(PageFaultAccess::Write, dirty_fault_address); + dirty_fault_complete.release(); + }); + const bool dirty_fault_bypassed_lane = + dirty_fault_complete.try_acquire_for(std::chrono::milliseconds(200)); + block_release.release(); + if (!dirty_fault_bypassed_lane) { + dirty_fault_complete.acquire(); + } + dirty_fault_thread.join(); + uint32_t dirty_fault_backing = 0; + std::memcpy(&dirty_fault_backing, + reinterpret_cast(dirty_fault_address), + sizeof(dirty_fault_backing)); + Require("GpuCommandLane", "dirty fault synchronization", + !dirty_fault_bypassed_lane && dirty_fault_handled && + dirty_fault_backing == dirty_fault_value && + !buffer_cache.HasGpuDirtyBytes(dirty_fault_address, + sizeof(dirty_fault_value)), + "a GPU-dirty write fault bypassed synchronization or lost the " + "published bytes"); + + constexpr uint64_t empty_copy_fault_address = fault_base + 0xe000; + constexpr uint32_t empty_copy_fault_value = 0x6b28d1efu; + gpu.SendCommandSync([&] { + auto dirty = buffer_cache.ObtainBuffer( + scheduler.Current(), empty_copy_fault_address, + sizeof(empty_copy_fault_value), true, false); + Require("GpuCommandLane", "empty-copy write setup", + dirty.owner != nullptr, + "empty-copy write test could not create a cached Buffer"); + scheduler.Current().RetainResourceUntilFence(dirty.owner); + buffer_cache.FillBuffer(empty_copy_fault_address, + sizeof(empty_copy_fault_value), + empty_copy_fault_value); + buffer_cache.ReadMemory(empty_copy_fault_address, + sizeof(empty_copy_fault_value)); + const bool gpu_owned = buffer_cache.IsRegionGpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool cpu_owned = buffer_cache.IsRegionCpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + Require("GpuCommandLane", "empty-copy read ownership", + !gpu_owned && !cpu_owned, + "readback did not leave clean ownership for the queued write " + "invalidation"); + buffer_cache.ReadMemory(empty_copy_fault_address, + sizeof(empty_copy_fault_value), true); + }); + uint32_t empty_copy_fault_backing = 0; + std::memcpy(&empty_copy_fault_backing, + reinterpret_cast(empty_copy_fault_address), + sizeof(empty_copy_fault_backing)); + const bool empty_copy_cpu_owned = buffer_cache.IsRegionCpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool empty_copy_gpu_owned = buffer_cache.IsRegionGpuModified( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + const bool empty_copy_dirty = buffer_cache.HasGpuDirtyBytes( + empty_copy_fault_address, sizeof(empty_copy_fault_value)); + Require("GpuCommandLane", "empty-copy write ownership", + empty_copy_fault_backing == empty_copy_fault_value && + empty_copy_cpu_owned && !empty_copy_gpu_owned && + !empty_copy_dirty, + "write invalidation without a remaining copy did not establish CPU " + "ownership"); + + resources.UnmapMemory(fault_base, fault_size); + Require("GpuCommandLane", "processor fault unmap", + Libs::LibKernel::Memory::KernelMunmap(fault_base, fault_size) == 0, + "processor-fault direct-memory mapping release failed"); + Require("GpuCommandLane", "processor fault free", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + fault_direct_offset, fault_size) == 0, + "processor-fault direct-memory allocation release failed"); + + std::binary_semaphore command_entered{0}; + std::binary_semaphore release_command{0}; + std::atomic shutdown_complete{false}; + std::atomic submission_lane_entered{false}; + gpu.SendCommand([&] { + command_entered.release(); + release_command.acquire(); + gpu.Done(); + submission_lane_entered = true; + }); + command_entered.acquire(); + std::jthread shutdown_thread([&] { + gpu.Shutdown(); + shutdown_complete = true; + }); + while (!gpu.IsStopping()) { + std::this_thread::yield(); + } + Require("GpuCommandLane", "owned shutdown", !shutdown_complete.load(), + "GPU owner returned before an in-flight command completed"); + release_command.release(); + shutdown_thread.join(); + Require( + "GpuCommandLane", "owned shutdown completion", + shutdown_complete.load() && submission_lane_entered.load(), + "GPU owner did not drain a command that entered the submission lane"); + context.ShutdownGpu(); + std::printf("[host] %-32s ok\n", "GpuCommandLane"); + } + + void CheckUnifiedImageViewCache() { + EnsureRuntimeContext(); + constexpr const char *name = "UnifiedImageViewCache"; + CommandScheduler scheduler(Renderer(), m_runtime_context); + + ImageInfo color_info{}; + color_info.pixel_format = vk::Format::eR8G8B8A8Unorm; + color_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; + color_info.type = Prospero::ImageType::kColor2D; + color_info.extent = {8, 8, 1}; + color_info.resources = {2, 2}; + color_info.pitch = 8; + color_info.bytes_per_block = 4; + color_info.samples = 1; + color_info.tile_mode = Prospero::TileMode::kLinear; + color_info.mip_layout[0] = {0, 512, 8, 8}; + color_info.mip_layout[1] = {512, 128, 4, 4}; + + Libs::Graphics::Image color(m_runtime_context, scheduler, color_info); + const vk::ComponentMapping identity{}; + const vk::ComponentMapping bgra{ + vk::ComponentSwizzle::eB, vk::ComponentSwizzle::eG, + vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eA}; + + ImageViewInfo sampled{}; + sampled.format = color_info.pixel_format; + sampled.type = vk::ImageViewType::e2D; + sampled.aspect = vk::ImageAspectFlagBits::eColor; + sampled.base_level = 0; + sampled.level_count = 1; + sampled.base_layer = 0; + sampled.layer_count = 1; + sampled.mapping = identity; + sampled.usage = vk::ImageUsageFlagBits::eSampled; + + const auto first = color.FindView(sampled); + const auto first_again = color.FindView(sampled); + auto swizzled_info = sampled; + swizzled_info.mapping = bgra; + const auto swizzled = color.FindView(swizzled_info); + auto mip_info = sampled; + mip_info.base_level = 1; + const auto mip = color.FindView(mip_info); + auto layer_info = sampled; + layer_info.base_layer = 1; + const auto layer = color.FindView(layer_info); + auto array_info = sampled; + array_info.type = vk::ImageViewType::e2DArray; + array_info.layer_count = 2; + const auto array = color.FindView(array_info); + auto reinterpreted_info = sampled; + reinterpreted_info.format = vk::Format::eR8G8B8A8Uint; + const auto reinterpreted = color.FindView(reinterpreted_info); + auto storage_info = sampled; + storage_info.usage = vk::ImageUsageFlagBits::eStorage; + const auto storage = color.FindView(storage_info); + auto attachment_info = sampled; + attachment_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + const auto attachment = color.FindView(attachment_info); + + Require(name, "color views", + first != nullptr && first_again == first && swizzled != nullptr && + swizzled != first && mip != nullptr && mip != first && + layer != nullptr && layer != first && array != nullptr && + array != first && reinterpreted != nullptr && + reinterpreted != first && storage != nullptr && + storage != first && attachment == first && + color.views.views.size() == 7, + "dynamic view identity omitted mapping, format, mip, layer, type, " + "or storage usage"); + + ImageInfo depth_info{}; + depth_info.pixel_format = vk::Format::eD32Sfloat; + depth_info.guest_format = Prospero::BufferFormat::k32Float; + depth_info.type = Prospero::ImageType::kColor2D; + depth_info.extent = {8, 8, 1}; + depth_info.resources = {1, 2}; + depth_info.pitch = 8; + depth_info.bytes_per_block = 4; + depth_info.samples = 1; + depth_info.tile_mode = Prospero::TileMode::kDepth; + depth_info.mip_layout[0] = {0, 512, 8, 8}; + + Libs::Graphics::Image depth(m_runtime_context, scheduler, depth_info); + ImageViewInfo depth_sampled{}; + depth_sampled.format = depth_info.pixel_format; + depth_sampled.type = vk::ImageViewType::e2D; + depth_sampled.aspect = vk::ImageAspectFlagBits::eDepth; + depth_sampled.base_level = 0; + depth_sampled.level_count = 1; + depth_sampled.base_layer = 0; + depth_sampled.layer_count = 1; + depth_sampled.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR}; + depth_sampled.usage = vk::ImageUsageFlagBits::eSampled; + const auto depth_first = depth.FindView(depth_sampled); + auto depth_alias_info = depth_sampled; + depth_alias_info.format = vk::Format::eR32Uint; + const auto depth_alias = depth.FindView(depth_alias_info); + const auto depth_again = depth.FindView(depth_sampled); + auto depth_array_info = depth_sampled; + depth_array_info.type = vk::ImageViewType::e2DArray; + depth_array_info.layer_count = 2; + const auto depth_array = depth.FindView(depth_array_info); + auto depth_attachment_info = depth_array_info; + depth_attachment_info.mapping = {}; + depth_attachment_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto depth_attachment = depth.FindView(depth_attachment_info); + Require(name, "depth views", + depth_first != nullptr && depth_again == depth_first && + depth_alias == depth_first && depth_array != nullptr && + depth_array != depth_first && depth_attachment != nullptr && + depth_attachment != depth_array && + depth.views.views.size() == 3, + "unified depth view cache lost sampled/attachment identity"); + Require(name, "role-free backing", + color.backing.image_type == vk::ImageType::e2D && + color.backing.layers == 2 && + depth.backing.image_type == vk::ImageType::e2D && + ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Uint) && + !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat, + vk::Format::eR32Sfloat), + "role-free image backing or host format compatibility diverged"); + + auto line_info = color_info; + line_info.type = Prospero::ImageType::kColor1D; + line_info.extent = {8, 1, 1}; + line_info.resources = {2, 2}; + line_info.mip_layout[0] = {0, 64, 8, 1}; + line_info.mip_layout[1] = {64, 16, 4, 1}; + Libs::Graphics::Image line(m_runtime_context, scheduler, line_info); + auto line_view_info = sampled; + line_view_info.type = vk::ImageViewType::e1D; + const auto line_view = line.FindView(line_view_info); + auto line_array_info = line_view_info; + line_array_info.type = vk::ImageViewType::e1DArray; + line_array_info.layer_count = 2; + const auto line_array = line.FindView(line_array_info); + Require(name, "1D views", + line_view != nullptr && line_array != nullptr && + line_array != line_view && + line.backing.image_type == vk::ImageType::e1D && + line.backing.layers == 2, + "1D and 1D-array views did not use a first-class 1D backing"); + + auto volume_info = color_info; + volume_info.type = Prospero::ImageType::kColor3D; + volume_info.extent = {8, 8, 4}; + volume_info.resources = {2, 1}; + Libs::Graphics::Image volume(m_runtime_context, scheduler, volume_info); + auto slice_info = sampled; + slice_info.type = vk::ImageViewType::e2D; + slice_info.base_level = 1; + slice_info.base_layer = 1; + const auto slice = volume.FindView(slice_info); + auto slice_array_info = slice_info; + slice_array_info.type = vk::ImageViewType::e2DArray; + slice_array_info.base_layer = 0; + slice_array_info.layer_count = 2; + const auto slice_array = volume.FindView(slice_array_info); + Require(name, "3D slice views", + slice != nullptr && slice_array != nullptr && + slice_array != slice && + static_cast(volume.backing.flags & + vk::ImageCreateFlagBits::e2DArrayCompatible), + "2D slice views of a compatible 3D backing were rejected"); + std::printf("[host] %-32s ok\n", name); + } + + void CheckBufferCacheDirtyGarbageCollection() { + constexpr const char *name = "BufferCacheDirtyGarbageCollection"; + constexpr uintptr_t base = 0x0000000200700000ull; + constexpr uint64_t allocation_size = 0x2400000; + constexpr uint64_t allocation_alignment = 0x10000; + constexpr uint64_t first_offset = 0x100; + constexpr uint64_t second_offset = 0x200; + constexpr uint64_t clean_offset = 0x300; + constexpr uint64_t unmap_offset = 0x4100; + constexpr uint64_t partial_unmap_offset = 0x8100; + constexpr uint64_t partial_unmap_survivor_offset = 0xc100; + constexpr uint64_t ring_fault_first_offset = 0x2100; + constexpr uint64_t ring_fault_second_offset = 0x2110; + constexpr uint32_t first_value = 0x10203040u; + constexpr uint32_t second_value = 0x50607080u; + constexpr uint32_t first_stale = 0x0badf00du; + constexpr uint32_t second_stale = 0xdeadbeefu; + constexpr uint32_t clean_value = 0xaabbccddu; + constexpr uint32_t unmap_value = 0x91a2b3c4u; + constexpr uint32_t partial_unmap_value = 0x62738495u; + constexpr uint32_t partial_unmap_survivor_value = 0xa6b7c8d9u; + constexpr uint32_t ring_fault_first_value = 0x0a1b2c3du; + constexpr uint32_t ring_fault_second_value = 0x4e5f6071u; + + EnsureRuntimeContext(); + auto &context = Renderer(); + CommandScheduler scheduler(context, m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + context.InitializeGpu(nullptr); + auto &gpu = context.GetGpu(); + + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "dirty-GC direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "dirty-GC fixed direct-memory mapping failed"); + auto *memory = static_cast(mapped); + std::memcpy(memory + first_offset, &first_stale, sizeof(first_stale)); + std::memcpy(memory + second_offset, &second_stale, sizeof(second_stale)); + std::memcpy(memory + clean_offset, &clean_value, sizeof(clean_value)); + + { + GpuResourceManager resources(m_runtime_context, scheduler); + resources.SetGpu(&gpu); + auto &cache = resources.GetBufferCache(); + resources.MapMemory(base, allocation_size); + + const auto MarkGpuWrite = [&](uint64_t address, uint64_t size) { + auto allocation = + cache.ObtainBuffer(scheduler.Current(), address, size, true, false); + Require(name, "dirty allocation", allocation.owner != nullptr, + "dirty-GC buffer allocation failed"); + scheduler.Current().RetainResourceUntilFence(allocation.owner); + }; + + constexpr uint64_t index_offset = 0x180000; + constexpr uint64_t index_page = BufferCache::CACHING_PAGE_SIZE; + constexpr uint64_t index_span = 3 * index_page; + const uint64_t index_begin = base + index_offset; + Require(name, "registered-range empty lookup", + !cache.IsRegionRegistered(index_begin, index_span), + "empty Buffer cache reported a registered range"); + auto index_left = + cache.ObtainBuffer(scheduler.Current(), index_begin + 0x100, + sizeof(uint32_t), false, false); + auto index_right = cache.ObtainBuffer( + scheduler.Current(), index_begin + 2 * index_page + 0x100, + sizeof(uint32_t), false, false); + Require(name, "registered-range owners", + index_left.owner != nullptr && index_right.owner != nullptr, + "failed to create disjoint Buffer index owners"); + scheduler.Current().RetainResourceUntilFence(index_left.owner); + scheduler.Current().RetainResourceUntilFence(index_right.owner); + Require( + name, "registered-range boundaries", + cache.IsRegionRegistered(index_begin, 1) && + cache.IsRegionRegistered(index_begin + index_page - 1, 1) && + !cache.IsRegionRegistered(index_begin - 1, 1) && + !cache.IsRegionRegistered(index_begin + index_page, index_page) && + cache.IsRegionRegistered(index_begin + index_page, + index_page + 1) && + !cache.IsRegionRegistered(index_begin + index_span, 1) && + cache.IsRegionRegistered(index_begin - 1, index_span + 2), + "indexed lookup mishandled a half-open boundary, gap, or broad " + "overlap"); + auto index_bridge = + cache.ObtainBuffer(scheduler.Current(), index_begin + index_page - 1, + index_page + 2, false, false); + Require( + name, "registered-range merge", + index_bridge.owner != nullptr && + cache.IsRegionRegistered(index_begin + index_page, index_page), + "bridging acquisition did not publish its merged Buffer range"); + scheduler.Current().RetainResourceUntilFence(index_bridge.owner); + cache.UnmapMemory(index_begin, index_span); + Require(name, "registered-range removal", + !cache.IsRegionRegistered(index_begin, index_span), + "unmapped Buffer owner remained in the registered-range index"); + + MarkGpuWrite(base + first_offset, sizeof(first_value)); + MarkGpuWrite(base + second_offset, sizeof(second_value)); + cache.FillBuffer(base + first_offset, sizeof(first_value), first_value); + cache.FillBuffer(base + second_offset, sizeof(second_value), + second_value); + auto &download = BufferCacheTestAccess::DownloadBuffer(cache); + auto *fixed_download = &download; + const auto fixed_download_handle = download.Handle(); + auto [ring_tail, ring_tail_offset] = + download.Map(download.Size() - sizeof(uint32_t), 4); + Require(name, "fault-ring tail reservation", + ring_tail != nullptr && ring_tail_offset == 0, + "failed to position the reusable download ring near wrap"); + download.Commit(); + MarkGpuWrite(base + ring_fault_first_offset, + sizeof(ring_fault_first_value)); + MarkGpuWrite(base + ring_fault_second_offset, + sizeof(ring_fault_second_value)); + cache.FillBuffer(base + ring_fault_first_offset, + sizeof(ring_fault_first_value), ring_fault_first_value); + cache.FillBuffer(base + ring_fault_second_offset, + sizeof(ring_fault_second_value), + ring_fault_second_value); + Require(name, "fault-ring wrapped batch", + resources.HandleFault(PageFaultAccess::Read, + base + ring_fault_first_offset), + "fault readback could not wrap a live download-ring tick"); + uint64_t expected_packing_offset = 128; + uint64_t expected_packing_alignment = 1; + if (!download.IsCoherent()) { + Require(name, "fault-ring atom policy", + StreamBufferTestAccess::NormalizeReservation( + false, + m_runtime_context.physical_device_properties.limits + .nonCoherentAtomSize, + expected_packing_offset, expected_packing_alignment), + "failed to calculate non-coherent fault-ring stride"); + } + auto [packing_probe, packing_offset] = download.Map(1, 1, false); + Require(name, "adjacent download reservation stride", + packing_probe != nullptr && + packing_offset == expected_packing_offset, + "adjacent fault downloads did not reserve an atom-safe stride"); + download.Commit(); + uint32_t ring_fault_first_backing = 0; + uint32_t ring_fault_second_backing = 0; + std::memcpy(&ring_fault_first_backing, memory + ring_fault_first_offset, + sizeof(ring_fault_first_backing)); + std::memcpy(&ring_fault_second_backing, memory + ring_fault_second_offset, + sizeof(ring_fault_second_backing)); + Require(name, "fault-ring wrapped contents", + ring_fault_first_backing == ring_fault_first_value && + ring_fault_second_backing == ring_fault_second_value && + &BufferCacheTestAccess::DownloadBuffer(cache) == + fixed_download && + download.Handle() == fixed_download_handle && + download.Size() == (32ull << 20), + "wrapped fault batch published incorrect disjoint ranges"); + + for (uint32_t tick = 0; tick < 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "age before pressure", + cache.IsRegionRegistered(base, allocation_size), + "buffer was reclaimed without memory pressure"); + BufferCacheTestAccess::SetGarbageCollectionThresholds( + cache, 0, std::numeric_limits::max()); + const auto gc_submission_tick = scheduler.CurrentTick(); + cache.RunGarbageCollector(); + Require(name, "dirty retirement", + !cache.IsRegionRegistered(base, allocation_size), + "aged GPU-dirty buffer survived pressured " + "collection"); + + uint32_t first_before_completion = 0; + uint32_t second_before_completion = 0; + Libs::LibKernel::Memory::TryReadBacking(base + first_offset, + &first_before_completion, + sizeof(first_before_completion)); + Libs::LibKernel::Memory::TryReadBacking(base + second_offset, + &second_before_completion, + sizeof(second_before_completion)); + Require(name, "deferred dirty retirement", + first_before_completion == first_stale && + second_before_completion == second_stale && + cache.IsRegionGpuModified(base + first_offset, + sizeof(first_value)) && + cache.IsRegionGpuModified(base + second_offset, + sizeof(second_value)) && + cache.HasGpuDirtyBytes(base + first_offset, + sizeof(first_value)) && + cache.HasGpuDirtyBytes(base + second_offset, + sizeof(second_value)) && + scheduler.CurrentTick() == gc_submission_tick, + "buffer GC published bytes or cleared dirty " + "ownership before GPU completion"); + scheduler.FinishCurrent(); + + uint32_t first_backing = 0; + uint32_t second_backing = 0; + uint32_t clean_backing = 0; + std::memcpy(&first_backing, memory + first_offset, sizeof(first_backing)); + std::memcpy(&second_backing, memory + second_offset, + sizeof(second_backing)); + std::memcpy(&clean_backing, memory + clean_offset, sizeof(clean_backing)); + Require(name, "downloaded contents", + first_backing == first_value && second_backing == second_value && + clean_backing == clean_value && + !cache.IsRegionGpuModified(base + first_offset, + sizeof(first_value)) && + !cache.IsRegionGpuModified(base + second_offset, + sizeof(second_value)) && + !cache.HasGpuDirtyBytes(base + first_offset, + sizeof(first_value)) && + !cache.HasGpuDirtyBytes(base + second_offset, + sizeof(second_value)), + "dirty GC did not publish exact ranges before " + "clearing broad page ownership"); + + auto unmap_allocation = + cache.ObtainBuffer(scheduler.Current(), base + unmap_offset, + sizeof(unmap_value), true, false); + Require(name, "direct-unmap allocation", + unmap_allocation.owner != nullptr, + "direct-unmap buffer allocation failed"); + scheduler.Current().RetainResourceUntilFence(unmap_allocation.owner); + cache.FillBuffer(base + unmap_offset, sizeof(unmap_value), unmap_value); + cache.UnmapMemory(base + 0x4000, 0x4000); + uint32_t unmap_backing = 0; + std::memcpy(&unmap_backing, memory + unmap_offset, sizeof(unmap_backing)); + Require(name, "direct-unmap contents", + unmap_backing == unmap_value && + !cache.IsRegionRegistered(base + 0x4000, 0x4000), + "direct dirty unmap cleared tracking before " + "publishing bytes"); + + auto partial_unmap_allocation = cache.ObtainBuffer( + scheduler.Current(), base + 0x8000, 0x8000, true, false); + Require(name, "partial-unmap allocation", + partial_unmap_allocation.owner != nullptr, + "cross-page cached buffer allocation failed"); + scheduler.Current().RetainResourceUntilFence( + partial_unmap_allocation.owner); + cache.FillBuffer(base + partial_unmap_offset, sizeof(partial_unmap_value), + partial_unmap_value); + cache.FillBuffer(base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), + partial_unmap_survivor_value); + cache.UnmapMemory(base + 0x8000, 0x4000); + auto survivor = cache.ObtainBuffer( + scheduler.Current(), base + partial_unmap_survivor_offset, + sizeof(partial_unmap_survivor_value), false, true); + Require(name, "partial-unmap survivor", survivor.buffer != nullptr, + "still-mapped cached-buffer remainder could not be recreated"); + if (survivor.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence(survivor.owner); + } + auto partial_unmap_readback = + CreateHostBuffer(name, sizeof(partial_unmap_survivor_value), + vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy survivor_copy{survivor.offset, 0, + sizeof(partial_unmap_survivor_value)}; + scheduler.Current().Handle().copyBuffer( + survivor.buffer, partial_unmap_readback.buffer, 1, &survivor_copy); + vk::BufferMemoryBarrier survivor_barrier{}; + survivor_barrier.sType = vk::StructureType::eBufferMemoryBarrier; + survivor_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + survivor_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + survivor_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + survivor_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + survivor_barrier.buffer = partial_unmap_readback.buffer; + survivor_barrier.size = partial_unmap_readback.size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &survivor_barrier, 0, nullptr); + scheduler.Finish(); + Require(name, "partial-unmap survivor contents", + ReadBuffer(name, partial_unmap_readback, 1) == + std::vector{partial_unmap_survivor_value}, + "partial unmap recreated its still-mapped remainder from " + "uninitialized native bytes"); + DestroyBuffer(&partial_unmap_readback); + + constexpr uint64_t large_offset = 0x10000; + constexpr uint64_t large_size = 31ull * 1024 * 1024; + constexpr uint32_t large_value = 0x5aa55aa5u; + constexpr uint32_t large_stale = 0x12345678u; + std::memcpy(memory + large_offset, &large_stale, sizeof(large_stale)); + std::memcpy(memory + large_offset + large_size - sizeof(large_stale), + &large_stale, sizeof(large_stale)); + auto large_allocation = cache.ObtainBuffer( + scheduler.Current(), base + large_offset, large_size, true, false); + Require(name, "near-capacity dirty allocation", + large_allocation.owner != nullptr, + "failed to allocate the near-capacity dirty native buffer"); + scheduler.Current().RetainResourceUntilFence(large_allocation.owner); + cache.FillBuffer(base + large_offset, large_size, large_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "near-capacity deferred retirement", + !cache.IsRegionRegistered(base + large_offset, large_size), + "near-capacity dirty Buffer survived pressured collection"); + uint32_t large_before_completion = 0; + Libs::LibKernel::Memory::TryReadBacking(base + large_offset, + &large_before_completion, + sizeof(large_before_completion)); + Require(name, "near-capacity backing remained deferred", + large_before_completion == large_stale, + "near-capacity Buffer GC published before its scheduler tick"); + scheduler.FinishCurrent(); + const auto large_image_source = + cache.ObtainBufferForImage(base + large_offset, sizeof(large_value)); + Require(name, "fixed download after Buffer retirement", + large_image_source.buffer != nullptr && + &BufferCacheTestAccess::DownloadBuffer(cache) == + fixed_download && + fixed_download->Handle() == fixed_download_handle && + fixed_download->Size() == (32ull << 20), + "image acquisition replaced the shared Buffer download stream"); + uint32_t large_first = 0; + uint32_t large_last = 0; + Libs::LibKernel::Memory::TryReadBacking(base + large_offset, &large_first, + sizeof(large_first)); + Libs::LibKernel::Memory::TryReadBacking(base + large_offset + large_size - + sizeof(large_last), + &large_last, sizeof(large_last)); + Require(name, "near-capacity Buffer publication contents", + large_first == large_value && large_last == large_value, + "near-capacity Buffer GC did not publish its complete transfer"); + + constexpr uint64_t grouped_first_offset = 0x10000; + constexpr uint64_t grouped_second_offset = 0x1200000; + constexpr uint64_t grouped_owner_size = 17ull * 1024 * 1024; + constexpr uint32_t grouped_first_value = 0x1122aabbu; + constexpr uint32_t grouped_second_value = 0x3344ccddu; + constexpr uint32_t grouped_stale = 0; + static_assert(grouped_owner_size < (32ull << 20) && + grouped_owner_size * 2 > (32ull << 20)); + Libs::LibKernel::Memory::WriteBacking( + base + grouped_first_offset, &grouped_stale, sizeof(grouped_stale)); + Libs::LibKernel::Memory::WriteBacking( + base + grouped_second_offset, &grouped_stale, sizeof(grouped_stale)); + auto grouped_first = + cache.ObtainBuffer(scheduler.Current(), base + grouped_first_offset, + grouped_owner_size, true, false); + auto grouped_second = + cache.ObtainBuffer(scheduler.Current(), base + grouped_second_offset, + grouped_owner_size, true, false); + Require(name, "per-owner GC allocation", + grouped_first.owner != nullptr && + grouped_second.owner != nullptr && + grouped_first.owner != grouped_second.owner, + "disjoint GC candidates merged into one Buffer owner"); + scheduler.Current().RetainResourceUntilFence(grouped_first.owner); + scheduler.Current().RetainResourceUntilFence(grouped_second.owner); + cache.FillBuffer(base + grouped_first_offset, grouped_owner_size, + grouped_first_value); + cache.FillBuffer(base + grouped_second_offset, grouped_owner_size, + grouped_second_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "per-owner fixed-ring retirement", + !cache.IsRegionRegistered(base + grouped_first_offset, + grouped_owner_size) && + !cache.IsRegionRegistered(base + grouped_second_offset, + grouped_owner_size) && + &BufferCacheTestAccess::DownloadBuffer(cache) == + fixed_download && + fixed_download->Handle() == fixed_download_handle, + "GC aggregated disjoint retirees or replaced the download ring"); + scheduler.FinishCurrent(); + uint32_t grouped_first_backing = 0; + uint32_t grouped_second_backing = 0; + Libs::LibKernel::Memory::TryReadBacking(base + grouped_first_offset, + &grouped_first_backing, + sizeof(grouped_first_backing)); + Libs::LibKernel::Memory::TryReadBacking(base + grouped_second_offset, + &grouped_second_backing, + sizeof(grouped_second_backing)); + Require(name, "per-owner fixed-ring contents", + grouped_first_backing == grouped_first_value && + grouped_second_backing == grouped_second_value, + "per-owner GC transfers lost data while wrapping the fixed ring"); + + constexpr uint64_t disjoint_owner_offset = 0x2140000; + constexpr uint64_t disjoint_owner_size = 0x8000; + constexpr uint64_t disjoint_dirty_offset = disjoint_owner_offset + 0x100; + constexpr uint64_t disjoint_new_owner_offset = + disjoint_owner_offset + 0x5000; + constexpr uint32_t disjoint_value = 0x89abcdefu; + constexpr uint32_t disjoint_stale = 0x76543210u; + std::memcpy(memory + disjoint_dirty_offset, &disjoint_stale, + sizeof(disjoint_stale)); + auto wide_owner = + cache.ObtainBuffer(scheduler.Current(), base + disjoint_owner_offset, + disjoint_owner_size, false, true); + Require(name, "disjoint retirement owner", wide_owner.owner != nullptr, + "failed to create the clean cross-page retirement owner"); + scheduler.Current().RetainResourceUntilFence(wide_owner.owner); + auto dirty_alias = + cache.ObtainBuffer(scheduler.Current(), base + disjoint_dirty_offset, + sizeof(disjoint_value), true, false); + if (dirty_alias.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence(dirty_alias.owner); + } + cache.FillBuffer(base + disjoint_dirty_offset, sizeof(disjoint_value), + disjoint_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "disjoint deferred retirement", + !cache.IsRegionRegistered(base + disjoint_owner_offset, + disjoint_owner_size), + "cross-page owner survived pressured collection"); + uint32_t disjoint_before_unmap = 0; + Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, + &disjoint_before_unmap, + sizeof(disjoint_before_unmap)); + Require( + name, "disjoint retirement remained deferred", + disjoint_before_unmap == disjoint_stale, + "whole-owner Buffer publication completed before synchronization"); + scheduler.FinishCurrent(); + cache.UnmapMemory(base + disjoint_owner_offset + 0x4000, 0x4000); + uint32_t disjoint_after_unmap = 0; + Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, + &disjoint_after_unmap, + sizeof(disjoint_after_unmap)); + Require(name, "disjoint synchronized unmap", + disjoint_after_unmap == disjoint_value && + !cache.HasGpuDirtyBytes(base + disjoint_dirty_offset, + sizeof(disjoint_value)), + "disjoint unmap lost the completed whole-owner publication"); + auto disjoint_new_owner = cache.ObtainBuffer( + scheduler.Current(), base + disjoint_new_owner_offset, + sizeof(disjoint_value), true, false); + Require(name, "disjoint post-publication reacquire", + disjoint_new_owner.owner != nullptr, + "disjoint acquisition failed after whole-owner publication"); + scheduler.Current().RetainResourceUntilFence(disjoint_new_owner.owner); + uint32_t disjoint_backing = 0; + Libs::LibKernel::Memory::TryReadBacking(base + disjoint_dirty_offset, + &disjoint_backing, + sizeof(disjoint_backing)); + Require(name, "disjoint retirement contents", + disjoint_backing == disjoint_value, + "old retirement callback lost dirty bytes or retained ownership " + "after a disjoint reacquire"); + cache.ReadMemory(base + disjoint_new_owner_offset, + sizeof(disjoint_value)); + + constexpr uint64_t reacquire_owner_offset = 0x2200000; + constexpr uint64_t reacquire_owner_size = 0x8000; + constexpr uint64_t reacquire_dirty_offset = + reacquire_owner_offset + 0x100; + constexpr uint64_t reacquire_disjoint_offset = + reacquire_owner_offset + 0x5000; + constexpr uint32_t reacquire_value = 0xc001d00du; + constexpr uint32_t reacquire_stale = 0x0badf00du; + std::memcpy(memory + reacquire_dirty_offset, &reacquire_stale, + sizeof(reacquire_stale)); + auto reacquire_owner = + cache.ObtainBuffer(scheduler.Current(), base + reacquire_owner_offset, + reacquire_owner_size, false, true); + Require(name, "reacquire retirement owner", + reacquire_owner.owner != nullptr, + "failed to create a fresh cross-page retirement owner"); + scheduler.Current().RetainResourceUntilFence(reacquire_owner.owner); + auto reacquire_dirty = + cache.ObtainBuffer(scheduler.Current(), base + reacquire_dirty_offset, + sizeof(reacquire_value), true, false); + if (reacquire_dirty.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence(reacquire_dirty.owner); + } + cache.FillBuffer(base + reacquire_dirty_offset, sizeof(reacquire_value), + reacquire_value); + for (uint32_t tick = 0; tick <= 160; tick++) { + cache.RunGarbageCollector(); + } + Require(name, "reacquire deferred retirement", + !cache.IsRegionRegistered(base + reacquire_owner_offset, + reacquire_owner_size), + "fresh cross-page owner survived pressured collection"); + uint32_t reacquire_before = 0; + Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, + &reacquire_before, + sizeof(reacquire_before)); + Require(name, "reacquire publication remained deferred", + reacquire_before == reacquire_stale, + "fresh whole-owner publication completed before reacquisition"); + scheduler.FinishCurrent(); + auto reacquired = cache.ObtainBuffer( + scheduler.Current(), base + reacquire_disjoint_offset, + sizeof(reacquire_value), true, false); + Require(name, "independent disjoint post-publication reacquire", + reacquired.owner != nullptr, + "clean disjoint-half acquisition failed after publication"); + scheduler.Current().RetainResourceUntilFence(reacquired.owner); + uint32_t reacquire_after = 0; + Libs::LibKernel::Memory::TryReadBacking(base + reacquire_dirty_offset, + &reacquire_after, + sizeof(reacquire_after)); + Require(name, "independent disjoint reacquire contents", + reacquire_after == reacquire_value && + !cache.HasGpuDirtyBytes(base + reacquire_dirty_offset, + sizeof(reacquire_value)), + "disjoint-half acquisition failed to publish the retired " + "owner's dirty prefix"); + cache.ReadMemory(base + reacquire_disjoint_offset, + sizeof(reacquire_value)); + + resources.SetGpu(nullptr); + resources.UnmapMemory(base, allocation_size); + scheduler.Finish(); + } + context.ShutdownGpu(); + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "dirty-GC direct-memory mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "dirty-GC direct-memory allocation release failed"); + std::printf("[host] %-32s ok\n", name); + } + + void CheckUnifiedTextureCacheFlow() { + constexpr const char *name = "UnifiedTextureCacheFlow"; + constexpr uintptr_t base = 0x0000000200600000ull; + constexpr uint64_t allocation_size = 0x2800000; + constexpr uint64_t allocation_alignment = 0x200000; + EnsureRuntimeContext(); + auto &context = Renderer(); + CommandScheduler scheduler(context, m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + context.InitializeGpu(nullptr); + auto &gpu = context.GetGpu(); + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "cache test direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "fixed cache direct-memory mapping failed"); + auto *memory = static_cast(mapped); + const uint32_t initial = 0x44332211u; + std::memcpy(memory, &initial, sizeof(initial)); + + { + GpuResourceManager resources(m_runtime_context, scheduler); + resources.SetGpu(&gpu); + auto &texture_cache = resources.GetTextureCache(); + const auto [narrow_download, narrow_download_offset] = + TextureCacheTestAccess::MapDownload(texture_cache, 4, 4); + const auto [wide_download, wide_download_offset] = + TextureCacheTestAccess::MapDownload(texture_cache, 1, 16); + Require(name, "image download alignment", + narrow_download != nullptr && narrow_download_offset % 4 == 0 && + wide_download != nullptr && wide_download_offset % 16 == 0, + "wide/block image readback was not aligned to its texel block"); + resources.MapMemory(base, allocation_size); + + ImageDesc sampled{}; + sampled.type = BindingType::Texture; + sampled.info.data = {base, sizeof(initial)}; + sampled.info.pixel_format = vk::Format::eR8G8B8A8Srgb; + sampled.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb; + sampled.info.type = Prospero::ImageType::kColor2D; + sampled.info.extent = {1, 1, 1}; + sampled.info.resources = {1, 1}; + sampled.info.pitch = 1; + sampled.info.bytes_per_block = 4; + sampled.info.samples = 1; + sampled.info.tile_mode = Prospero::TileMode::kLinear; + sampled.info.mip_layout[0] = {0, 4, 1, 1}; + sampled.view_info.format = sampled.info.pixel_format; + sampled.view_info.type = vk::ImageViewType::e2D; + sampled.view_info.aspect = vk::ImageAspectFlagBits::eColor; + sampled.view_info.usage = vk::ImageUsageFlagBits::eSampled; + + auto &command = scheduler.Current(); + auto first_desc = sampled; + const auto first = texture_cache.FindImage(first_desc); + auto repeated_desc = sampled; + const auto repeated = texture_cache.FindImage(repeated_desc); + auto compatible_desc = sampled; + compatible_desc.info.pixel_format = vk::Format::eR8G8B8A8Uint; + compatible_desc.info.guest_format = Prospero::BufferFormat::k8_8_8_8UInt; + compatible_desc.view_info.format = compatible_desc.info.pixel_format; + const auto compatible = texture_cache.FindImage(compatible_desc); + Require(name, "normalized FindImage", + first && repeated == first && compatible == first && + texture_cache.GetImage(first).info.pixel_format == + vk::Format::eR8G8B8A8Srgb, + "registered compatible backing did not reuse one ImageId"); + + const auto IsOnlyImage = [](const std::vector &ids, + ImageId expected) { + return ids.size() == 1 && ids.front() == expected; + }; + const auto exact_byte_miss = + TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, false); + const auto touched_page_hit = + TextureCacheTestAccess::FindImages(texture_cache, base + 8, 1, true); + const auto next_page_miss = TextureCacheTestAccess::FindImages( + texture_cache, base + 0x1000, 1, true); + Require(name, "exact and 4-KiB page filtering", + exact_byte_miss.empty() && IsOnlyImage(touched_page_hit, first) && + next_page_miss.empty(), + "coarse candidates did not preserve exact-byte and touched-page " + "semantics"); + + const auto MakeOwnershipInfo = [](uint64_t address, uint64_t size) { + ImageInfo info{}; + info.data = {address, size}; + info.extent = {1, 1, 1}; + info.resources = {1, 1}; + info.samples = 1; + return info; + }; + const auto spanning_info = MakeOwnershipInfo(base + 0x1ff000, 0x2000); + const auto spanning = + TextureCacheTestAccess::InsertImage(texture_cache, spanning_info); + const auto spanning_results = TextureCacheTestAccess::FindImages( + texture_cache, spanning_info.data.address, spanning_info.data.size, + false); + Require(name, "one-MiB cross-page deduplication", + spanning && IsOnlyImage(spanning_results, spanning) && + TextureCacheTestAccess::PageOwnerCount( + texture_cache, spanning_info.data.address) == 1 && + TextureCacheTestAccess::PageOwnerCount( + texture_cache, spanning_info.data.End() - 1) == 1, + "an image spanning two coarse pages was missing or returned more " + "than once"); + TextureCacheTestAccess::DeleteImage(texture_cache, spanning); + Require(name, "cross-page owner cleanup", + TextureCacheTestAccess::PageOwnerCount( + texture_cache, spanning_info.data.address) == 0 && + TextureCacheTestAccess::PageOwnerCount( + texture_cache, spanning_info.data.End() - 1) == 0 && + TextureCacheTestAccess::FindImages( + texture_cache, spanning_info.data.address, + spanning_info.data.size, false) + .empty(), + "cross-page unregister left a stale coarse-page membership"); + + const auto shared_info = MakeOwnershipInfo(base + 0x500100, 0x100); + const auto shared_first = + TextureCacheTestAccess::InsertImage(texture_cache, shared_info); + const auto shared_second = + TextureCacheTestAccess::InsertImage(texture_cache, shared_info); + const auto shared_results = TextureCacheTestAccess::FindImages( + texture_cache, shared_info.data.address, shared_info.data.size, + false); + Require(name, "shared coarse-page registration", + shared_results.size() == 2 && + std::find(shared_results.begin(), shared_results.end(), + shared_first) != shared_results.end() && + std::find(shared_results.begin(), shared_results.end(), + shared_second) != shared_results.end(), + "two registered owners were not retained in one coarse page"); + TextureCacheTestAccess::DeleteImage(texture_cache, shared_first); + const auto shared_survivor = TextureCacheTestAccess::FindImages( + texture_cache, shared_info.data.address, shared_info.data.size, + false); + Require(name, "shared coarse-page unregister", + IsOnlyImage(shared_survivor, shared_second) && + TextureCacheTestAccess::PageOwnerCount( + texture_cache, shared_info.data.address) == 1, + "unregistering one owner removed its coarse-page neighbor"); + TextureCacheTestAccess::DeleteImage(texture_cache, shared_second); + Require(name, "final shared coarse-page unregister", + TextureCacheTestAccess::PageOwnerCount( + texture_cache, shared_info.data.address) == 0, + "the final shared owner remained registered"); + + constexpr uint64_t large_owner_size = 64ull * 1024 * 1024; + const auto large_info = + MakeOwnershipInfo(base + 0x800000, large_owner_size); + const auto large_owner = + TextureCacheTestAccess::InsertImage(texture_cache, large_info); + Require(name, "production one-MiB registration granularity", + TextureCacheTestAccess::OwnedPageCount( + texture_cache, large_info.data.address, large_info.data.size, + large_owner) == 64, + "64 MiB image registration did not create exactly 64 coarse " + "memberships"); + TextureCacheTestAccess::DeleteImage(texture_cache, large_owner); + Require(name, "production one-MiB unregister granularity", + TextureCacheTestAccess::OwnedPageCount( + texture_cache, large_info.data.address, large_info.data.size, + large_owner) == 0, + "large image unregister left coarse memberships behind"); + + const ImageId stale{first.index, first.generation + 1}; + TextureCacheTestAccess::AddPageOwner(texture_cache, base, stale); + const auto stale_filtered = TextureCacheTestAccess::FindImages( + texture_cache, base, sizeof(initial), false); + Require(name, "stale page owner filtering", + IsOnlyImage(stale_filtered, first) && + TextureCacheTestAccess::RemovePageOwner(texture_cache, base, + stale), + "a stale generation escaped the direct page-owner lookup"); + + TextureCacheTestAccess::SetQueryEpoch(texture_cache, UINT32_MAX); + const auto wrap_results = TextureCacheTestAccess::FindImages( + texture_cache, base, sizeof(initial), false); + Require(name, "page-owner query epoch wrap", + IsOnlyImage(wrap_results, first) && + TextureCacheTestAccess::QueryEpoch(texture_cache) == 1, + "query deduplication failed while wrapping its epoch"); + + auto &image = texture_cache.GetImage(first); + constexpr uint32_t final_sampled_value = 0x88776655u; + Require(name, "sampled write between discovery and acquisition", + resources.HandleFault(PageFaultAccess::Write, + sampled.info.data.address), + "sampled image did not accept a CPU update after discovery"); + std::memcpy(memory, &final_sampled_value, sizeof(final_sampled_value)); + Require(name, "sampled final-acquisition precondition", + image.IsCpuDirty(), + "the between-phase CPU write did not dirty the discovered image"); + const auto first_view = texture_cache.FindTexture(first, first_desc); + const auto repeated_view = + texture_cache.FindTexture(repeated, repeated_desc); + auto mapped_view_info = sampled.view_info; + mapped_view_info.mapping.r = vk::ComponentSwizzle::eB; + mapped_view_info.mapping.b = vk::ComponentSwizzle::eR; + const auto mapped_view = image.FindView(mapped_view_info); + Require( + name, "dynamic views", + first_view != nullptr && repeated_view == first_view && + mapped_view != nullptr && mapped_view != first_view && + image.views.views.size() == 2 && !image.usage.texture && + !image.IsGpuModified() && !image.IsCpuDirty() && + !TextureCacheTestAccess::PendingDownload(texture_cache, first), + "TextureCache::FindTexture did not preserve sampled view " + "identity, final refresh, and cache-only ownership"); + + ImageInfo chain = sampled.info; + chain.data = {0x10000, 0x8000}; + chain.extent = {8, 8, 1}; + chain.resources = {2, 4}; + chain.pitch = 8; + chain.mip_layout[0] = {0, 0x4000, 8, 8}; + chain.mip_layout[1] = {0x4000, 0x4000, 4, 4}; + ImageInfo subresource = chain; + subresource.data = {0x16000, 0x1000}; + subresource.extent = {4, 4, 1}; + subresource.resources = {1, 1}; + subresource.pitch = 4; + subresource.mip_layout[0] = {0, 0x1000, 4, 4}; + const auto mip = subresource.MipOf(chain); + Require(name, "overlap resolution", + mip == 1 && subresource.SliceOf(chain, mip) == 2 && + ImageRangeOverlaps(chain.data, subresource.data) && + ImagePageRangesOverlap(chain.data, subresource.data), + "normalized mip/slice overlap did not resolve"); + + auto exact_desc = compatible_desc; + const auto exact = texture_cache.FindImage(exact_desc, true); + Require(name, "exact-format coexistence", + exact && exact != first && + TextureCacheTestAccess::Contains(texture_cache, first) && + texture_cache.GetImage(first).info.pixel_format == + sampled.info.pixel_format && + texture_cache.GetImage(exact).info.pixel_format == + compatible_desc.info.pixel_format, + "exact-format lookup replaced or synchronized its compatible " + "cache record"); + auto compatible_after_exact_desc = compatible_desc; + const auto compatible_after_exact = + texture_cache.FindImage(compatible_after_exact_desc); + Require(name, "last compatible format winner", + compatible_after_exact == exact, + "non-exact lookup did not retain the last compatible " + "registration"); + + auto null_desc = sampled; + null_desc.info.data = {}; + auto null_repeat_desc = null_desc; + const auto null_image = texture_cache.FindImage(null_desc); + const auto null_repeat = texture_cache.FindImage(null_repeat_desc); + Require(name, "null image", + null_image && null_repeat == null_image && null_image != exact, + "typed null-image lookup was not stable"); + + auto MakeLinearDesc = + [&](uint64_t address, uint64_t size, vk::Format format, + Prospero::BufferFormat guest_format, Prospero::ImageType type, + vk::Extent3D extent, uint32_t layers, uint32_t bytes_per_block, + uint32_t samples) { + ImageDesc desc{}; + desc.type = BindingType::Texture; + desc.info.data = {address, size}; + desc.info.pixel_format = format; + desc.info.guest_format = guest_format; + desc.info.type = type; + desc.info.extent = extent; + desc.info.resources = {1, layers}; + desc.info.pitch = extent.width; + desc.info.bytes_per_block = bytes_per_block; + desc.info.samples = samples; + desc.info.tile_mode = Prospero::TileMode::kLinear; + desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; + desc.view_info.format = format; + if (type == Prospero::ImageType::kColor3D) { + desc.view_info.type = vk::ImageViewType::e3D; + } else if (layers > 1) { + desc.view_info.type = vk::ImageViewType::e2DArray; + } else { + desc.view_info.type = vk::ImageViewType::e2D; + } + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.layer_count = layers; + desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + return desc; + }; + const auto HostReadBarrier = [&](vk::Buffer buffer, uint64_t size, + vk::PipelineStageFlags source_stage, + vk::AccessFlags source_access) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = source_access; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = size; + scheduler.Current().Handle().pipelineBarrier( + source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &barrier, 0, nullptr); + }; + const auto TransferReadBarrier = [&](vk::Buffer buffer, uint64_t size) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eTransferRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eTransfer, {}, 0, nullptr, 1, &barrier, + 0, nullptr); + }; + + // A formatted Buffer read must use the private // shadPS4-shaped image-copy path. Use a request larger - // than the stream shortcut and poison guest backing - // after upload so stale CPU staging cannot accidentally - // satisfy the content check. - constexpr uint64_t mip_prefix_offset = 0x2740000; - constexpr auto mip_format = Prospero::BufferFormat::k32UInt; - constexpr auto mip_tile = Prospero::TileMode::kLinear; - constexpr uint32_t mip_width = 4097; - constexpr uint32_t mip_height = 1; - constexpr uint32_t mip_levels = 1; - const uint32_t mip_pitch = - TileGetTexturePitch(mip_format, mip_width, mip_tile); - TileSizeAlign mip_total {}; - std::array mip_sizes {}; - std::array mip_padded {}; - TileGetTextureSize(mip_format, mip_width, mip_height, mip_levels, mip_tile, - &mip_total, mip_sizes.data(), mip_padded.data()); - const uint64_t mip_prefix_size = mip_sizes[0].offset + mip_sizes[0].size; - const uint64_t mip_guest_size = mip_total.size + 256; - Require(name, "formatted mip fixture", - mip_sizes[0].offset == 0 && mip_prefix_size > BufferCache::CACHING_PAGE_SIZE && - mip_prefix_size < mip_guest_size && mip_guest_size % sizeof(uint32_t) == 0, - "single-mip fixture did not expose a cache-sized fitting backing prefix"); - std::vector mip_native(mip_guest_size / sizeof(uint32_t)); - std::iota(mip_native.begin(), mip_native.end(), 0x61000000u); - std::memcpy(memory + mip_prefix_offset, mip_native.data(), mip_guest_size); - auto mip_desc = MakeLinearDesc( - base + mip_prefix_offset, mip_guest_size, vk::Format::eR32Uint, mip_format, - Prospero::ImageType::kColor2D, {mip_width, mip_height, 1}, 1, sizeof(uint32_t), 1); - mip_desc.info.resources.levels = mip_levels; - mip_desc.info.pitch = mip_pitch; - mip_desc.view_info.level_count = mip_levels; - for (uint32_t level = 0; level < mip_levels; level++) { - mip_desc.info.mip_layout[level] = {mip_sizes[level].offset, mip_sizes[level].size, - mip_padded[level].width, - mip_padded[level].height}; - } - const auto mip_image = texture_cache.FindImage(mip_desc); - (void)texture_cache.FindTexture(mip_image, mip_desc); - texture_cache.MarkGpuWritten(mip_image); - std::vector mip_stale(mip_native.size(), 0xdeadbeefu); - Libs::LibKernel::Memory::WriteBacking(base + mip_prefix_offset, mip_stale.data(), - mip_guest_size); + // than the stream shortcut and poison guest backing + // after upload so stale CPU staging cannot accidentally + // satisfy the content check. + constexpr uint64_t mip_prefix_offset = 0x2740000; + constexpr auto mip_format = Prospero::BufferFormat::k32UInt; + constexpr auto mip_tile = Prospero::TileMode::kLinear; + constexpr uint32_t mip_width = 4097; + constexpr uint32_t mip_height = 1; + constexpr uint32_t mip_levels = 1; + const uint32_t mip_pitch = + TileGetTexturePitch(mip_format, mip_width, mip_tile); + TileSizeAlign mip_total{}; + std::array mip_sizes{}; + std::array mip_padded{}; + TileGetTextureSize(mip_format, mip_width, mip_height, mip_levels, + mip_tile, &mip_total, mip_sizes.data(), + mip_padded.data()); + const uint64_t mip_prefix_size = mip_sizes[0].offset + mip_sizes[0].size; + const uint64_t mip_guest_size = mip_total.size + 256; + Require(name, "formatted mip fixture", + mip_sizes[0].offset == 0 && + mip_prefix_size > BufferCache::CACHING_PAGE_SIZE && + mip_prefix_size < mip_guest_size && + mip_guest_size % sizeof(uint32_t) == 0, + "single-mip fixture did not expose a cache-sized fitting backing " + "prefix"); + std::vector mip_native(mip_guest_size / sizeof(uint32_t)); + std::iota(mip_native.begin(), mip_native.end(), 0x61000000u); + std::memcpy(memory + mip_prefix_offset, mip_native.data(), + mip_guest_size); + auto mip_desc = MakeLinearDesc( + base + mip_prefix_offset, mip_guest_size, vk::Format::eR32Uint, + mip_format, Prospero::ImageType::kColor2D, {mip_width, mip_height, 1}, + 1, sizeof(uint32_t), 1); + mip_desc.info.resources.levels = mip_levels; + mip_desc.info.pitch = mip_pitch; + mip_desc.view_info.level_count = mip_levels; + for (uint32_t level = 0; level < mip_levels; level++) { + mip_desc.info.mip_layout[level] = { + mip_sizes[level].offset, mip_sizes[level].size, + mip_padded[level].width, mip_padded[level].height}; + } + const auto mip_image = texture_cache.FindImage(mip_desc); + (void)texture_cache.FindTexture(mip_image, mip_desc); + texture_cache.MarkGpuWritten(mip_image); + std::vector mip_stale(mip_native.size(), 0xdeadbeefu); + Libs::LibKernel::Memory::WriteBacking(base + mip_prefix_offset, + mip_stale.data(), mip_guest_size); - Libs::Graphics::Buffer mip_insufficient( - m_runtime_context, scheduler, MemoryUsage::DeviceLocal, mip_desc.info.data.address, - AllFlags, mip_prefix_size - 1); - Libs::Graphics::Buffer mip_prefix(m_runtime_context, scheduler, - MemoryUsage::DeviceLocal, mip_desc.info.data.address, - AllFlags, mip_prefix_size); - Require(name, "formatted mip containment", - !BufferCacheTestAccess::SynchronizeBufferFromImage( - resources.GetBufferCache(), mip_insufficient, mip_desc.info.data.address, - mip_prefix_size - 1) && - BufferCacheTestAccess::SynchronizeBufferFromImage( - resources.GetBufferCache(), mip_prefix, mip_desc.info.data.address, - mip_prefix_size) && - texture_cache.GetImage(mip_image).IsGpuModified(), - "image synchronization accepted a partial first " - "mip, rejected a fitting mip " - "prefix, or transferred ownership"); + Libs::Graphics::Buffer mip_insufficient( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + mip_desc.info.data.address, AllFlags, mip_prefix_size - 1); + Libs::Graphics::Buffer mip_prefix( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + mip_desc.info.data.address, AllFlags, mip_prefix_size); + Require(name, "formatted mip containment", + !BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), mip_insufficient, + mip_desc.info.data.address, mip_prefix_size - 1) && + BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), mip_prefix, + mip_desc.info.data.address, mip_prefix_size) && + texture_cache.GetImage(mip_image).IsGpuModified(), + "image synchronization accepted a partial first " + "mip, rejected a fitting mip " + "prefix, or transferred ownership"); - auto& buffer_cache = resources.GetBufferCache(); - BufferBinding mip_formatted; - bool formatted_owner_was_absent = false; - gpu.SendCommandSync([&] { - formatted_owner_was_absent = - !buffer_cache.IsRegionRegistered(mip_desc.info.data.address, - mip_desc.info.data.size); - mip_formatted = buffer_cache.ObtainBuffer(command, mip_desc.info.data.address, - mip_desc.info.data.size, false, true, true); - }); - Require(name, "formatted cache fixture", formatted_owner_was_absent, - "formatted image already had a cached Buffer owner"); - Require(name, "formatted Buffer path", - mip_formatted.owner != nullptr && mip_formatted.buffer != nullptr && - texture_cache.GetImage(mip_image).IsGpuModified(), - "formatted read bypassed the cached image-copy " - "path or transferred ownership"); - command.RetainResourceUntilFence(mip_formatted.owner); - auto mip_prefix_readback = - CreateHostBuffer(name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst, - std::vector(mip_prefix_size / sizeof(uint32_t), 0)); - auto mip_formatted_readback = - CreateHostBuffer(name, mip_guest_size, vk::BufferUsageFlagBits::eTransferDst, - std::vector(mip_guest_size / sizeof(uint32_t), 0)); - TransferReadBarrier(mip_prefix.Handle(), mip_prefix_size); - const vk::BufferCopy mip_prefix_copy {0, 0, mip_prefix_size}; - command.Handle().copyBuffer(mip_prefix.Handle(), mip_prefix_readback.buffer, 1, - &mip_prefix_copy); - TransferReadBarrier(mip_formatted.buffer, mip_guest_size); - const vk::BufferCopy mip_formatted_copy {mip_formatted.offset, 0, mip_guest_size}; - command.Handle().copyBuffer(mip_formatted.buffer, mip_formatted_readback.buffer, 1, - &mip_formatted_copy); - HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - HostReadBarrier(mip_formatted_readback.buffer, mip_formatted_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - const std::array volume_values {0x10203040u, 0x50607080u}; - std::memcpy(memory + 0x1000, volume_values.data(), sizeof(volume_values)); - auto array_desc = - MakeLinearDesc(base + 0x1000, sizeof(volume_values), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); - const auto array_image = texture_cache.FindImage(array_desc); - (void)texture_cache.FindTexture(array_image, array_desc); - texture_cache.MarkGpuWritten(array_image); - auto volume_desc = array_desc; - volume_desc.info.type = Prospero::ImageType::kColor3D; - volume_desc.info.extent = {1, 1, 2}; - volume_desc.info.resources.layers = 1; - volume_desc.view_info.type = vk::ImageViewType::e3D; - volume_desc.view_info.layer_count = 1; - const auto volume_image = texture_cache.FindImage(volume_desc); - Require(name, "2D-array/3D overlap copy", - volume_image && volume_image != array_image && - texture_cache.GetImage(volume_image).backing.image_type == - vk::ImageType::e3D && - texture_cache.GetImage(volume_image).IsGpuModified(), - "array-to-volume expansion lost slices or GPU " - "ownership"); + auto &buffer_cache = resources.GetBufferCache(); + BufferBinding mip_formatted; + bool formatted_owner_was_absent = false; + gpu.SendCommandSync([&] { + formatted_owner_was_absent = !buffer_cache.IsRegionRegistered( + mip_desc.info.data.address, mip_desc.info.data.size); + mip_formatted = buffer_cache.ObtainBuffer( + command, mip_desc.info.data.address, mip_desc.info.data.size, false, + true, true); + }); + Require(name, "formatted cache fixture", formatted_owner_was_absent, + "formatted image already had a cached Buffer owner"); + Require(name, "formatted Buffer path", + mip_formatted.owner != nullptr && + mip_formatted.buffer != nullptr && + texture_cache.GetImage(mip_image).IsGpuModified(), + "formatted read bypassed the cached image-copy " + "path or transferred ownership"); + command.RetainResourceUntilFence(mip_formatted.owner); + auto mip_prefix_readback = CreateHostBuffer( + name, mip_prefix_size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(mip_prefix_size / sizeof(uint32_t), 0)); + auto mip_formatted_readback = CreateHostBuffer( + name, mip_guest_size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(mip_guest_size / sizeof(uint32_t), 0)); + TransferReadBarrier(mip_prefix.Handle(), mip_prefix_size); + const vk::BufferCopy mip_prefix_copy{0, 0, mip_prefix_size}; + command.Handle().copyBuffer( + mip_prefix.Handle(), mip_prefix_readback.buffer, 1, &mip_prefix_copy); + TransferReadBarrier(mip_formatted.buffer, mip_guest_size); + const vk::BufferCopy mip_formatted_copy{mip_formatted.offset, 0, + mip_guest_size}; + command.Handle().copyBuffer(mip_formatted.buffer, + mip_formatted_readback.buffer, 1, + &mip_formatted_copy); + HostReadBarrier(mip_prefix_readback.buffer, mip_prefix_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + HostReadBarrier(mip_formatted_readback.buffer, + mip_formatted_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + const std::array volume_values{0x10203040u, 0x50607080u}; + std::memcpy(memory + 0x1000, volume_values.data(), sizeof(volume_values)); + auto array_desc = + MakeLinearDesc(base + 0x1000, sizeof(volume_values), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); + const auto array_image = texture_cache.FindImage(array_desc); + (void)texture_cache.FindTexture(array_image, array_desc); + texture_cache.MarkGpuWritten(array_image); + auto volume_desc = array_desc; + volume_desc.info.type = Prospero::ImageType::kColor3D; + volume_desc.info.extent = {1, 1, 2}; + volume_desc.info.resources.layers = 1; + volume_desc.view_info.type = vk::ImageViewType::e3D; + volume_desc.view_info.layer_count = 1; + const auto volume_image = texture_cache.FindImage(volume_desc); + Require(name, "2D-array/3D overlap copy", + volume_image && volume_image != array_image && + texture_cache.GetImage(volume_image).backing.image_type == + vk::ImageType::e3D && + texture_cache.GetImage(volume_image).IsGpuModified(), + "array-to-volume expansion lost slices or GPU " + "ownership"); - constexpr uint64_t unique_volume_offset = 0x27b0000; - std::memcpy(memory + unique_volume_offset, volume_values.data(), sizeof(volume_values)); - auto unique_volume_desc = MakeLinearDesc( - base + unique_volume_offset, sizeof(volume_values), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor3D, {1, 1, 2}, 1, sizeof(uint32_t), 1); - const auto unique_volume_image = texture_cache.FindImage(unique_volume_desc); - (void)texture_cache.FindTexture(unique_volume_image, unique_volume_desc); - texture_cache.MarkGpuWritten(unique_volume_image); - Libs::Graphics::Buffer partial_volume( - m_runtime_context, scheduler, MemoryUsage::DeviceLocal, - unique_volume_desc.info.data.address, AllFlags, - unique_volume_desc.info.data.size - sizeof(uint32_t)); - Libs::Graphics::Buffer full_volume( - m_runtime_context, scheduler, MemoryUsage::DeviceLocal, - unique_volume_desc.info.data.address, AllFlags, unique_volume_desc.info.data.size); - Require(name, "formatted volume containment", - !BufferCacheTestAccess::SynchronizeBufferFromImage( - resources.GetBufferCache(), partial_volume, - unique_volume_desc.info.data.address, - unique_volume_desc.info.data.size - sizeof(uint32_t)) && - BufferCacheTestAccess::SynchronizeBufferFromImage( - resources.GetBufferCache(), full_volume, - unique_volume_desc.info.data.address, - unique_volume_desc.info.data.size) && - texture_cache.GetImage(unique_volume_image).IsGpuModified(), - "partial 3D synchronization was accepted, " - "full-volume synchronization was " - "rejected, or ownership changed"); + constexpr uint64_t unique_volume_offset = 0x27b0000; + std::memcpy(memory + unique_volume_offset, volume_values.data(), + sizeof(volume_values)); + auto unique_volume_desc = MakeLinearDesc( + base + unique_volume_offset, sizeof(volume_values), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor3D, {1, 1, 2}, 1, sizeof(uint32_t), 1); + const auto unique_volume_image = + texture_cache.FindImage(unique_volume_desc); + (void)texture_cache.FindTexture(unique_volume_image, unique_volume_desc); + texture_cache.MarkGpuWritten(unique_volume_image); + Libs::Graphics::Buffer partial_volume( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + unique_volume_desc.info.data.address, AllFlags, + unique_volume_desc.info.data.size - sizeof(uint32_t)); + Libs::Graphics::Buffer full_volume( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + unique_volume_desc.info.data.address, AllFlags, + unique_volume_desc.info.data.size); + Require(name, "formatted volume containment", + !BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), partial_volume, + unique_volume_desc.info.data.address, + unique_volume_desc.info.data.size - sizeof(uint32_t)) && + BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), full_volume, + unique_volume_desc.info.data.address, + unique_volume_desc.info.data.size) && + texture_cache.GetImage(unique_volume_image).IsGpuModified(), + "partial 3D synchronization was accepted, " + "full-volume synchronization was " + "rejected, or ownership changed"); - constexpr uint64_t depth_containment_offset = 0x27c0000; - constexpr std::array depth_containment_values {0.25f, 0.75f, 0.5f}; - std::memcpy(memory + depth_containment_offset, depth_containment_values.data(), - sizeof(depth_containment_values)); - auto depth_containment = MakeLinearDesc( - base + depth_containment_offset, sizeof(depth_containment_values), - vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1); - depth_containment.type = BindingType::DepthTarget; - depth_containment.info.resources.levels = 2; - depth_containment.info.mip_layout[0] = {0, 2 * sizeof(float), 2, 1}; - depth_containment.info.mip_layout[1] = {2 * sizeof(float), sizeof(float), 1, 1}; - depth_containment.view_info.format = vk::Format::eD32Sfloat; - depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - depth_containment.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - depth_containment.view_info.level_count = 2; - const auto depth_containment_image = texture_cache.FindImage(depth_containment); - (void)texture_cache.FindDepthTarget(depth_containment_image, depth_containment); - texture_cache.MarkGpuWritten(depth_containment_image); - Libs::Graphics::Buffer partial_depth(m_runtime_context, scheduler, - MemoryUsage::DeviceLocal, - depth_containment.info.data.address, AllFlags, - depth_containment.info.mip_layout[0].size); - Require(name, "formatted depth containment", - !BufferCacheTestAccess::SynchronizeBufferFromImage( - resources.GetBufferCache(), partial_depth, - depth_containment.info.data.address, - depth_containment.info.mip_layout[0].size) && - texture_cache.GetImage(depth_containment_image).IsGpuModified(), - "partial depth synchronization was accepted or " - "transferred ownership"); + constexpr uint64_t depth_containment_offset = 0x27c0000; + constexpr std::array depth_containment_values{0.25f, 0.75f, + 0.5f}; + std::memcpy(memory + depth_containment_offset, + depth_containment_values.data(), + sizeof(depth_containment_values)); + auto depth_containment = MakeLinearDesc( + base + depth_containment_offset, sizeof(depth_containment_values), + vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, + Prospero::ImageType::kColor2D, {2, 1, 1}, 1, sizeof(float), 1); + depth_containment.type = BindingType::DepthTarget; + depth_containment.info.resources.levels = 2; + depth_containment.info.mip_layout[0] = {0, 2 * sizeof(float), 2, 1}; + depth_containment.info.mip_layout[1] = {2 * sizeof(float), sizeof(float), + 1, 1}; + depth_containment.view_info.format = vk::Format::eD32Sfloat; + depth_containment.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + depth_containment.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + depth_containment.view_info.level_count = 2; + const auto depth_containment_image = + texture_cache.FindImage(depth_containment); + (void)texture_cache.FindDepthTarget(depth_containment_image, + depth_containment); + texture_cache.MarkGpuWritten(depth_containment_image); + Libs::Graphics::Buffer partial_depth( + m_runtime_context, scheduler, MemoryUsage::DeviceLocal, + depth_containment.info.data.address, AllFlags, + depth_containment.info.mip_layout[0].size); + Require( + name, "formatted depth containment", + !BufferCacheTestAccess::SynchronizeBufferFromImage( + resources.GetBufferCache(), partial_depth, + depth_containment.info.data.address, + depth_containment.info.mip_layout[0].size) && + texture_cache.GetImage(depth_containment_image).IsGpuModified(), + "partial depth synchronization was accepted or " + "transferred ownership"); - auto native_array_info = array_desc.info; - native_array_info.data = {}; - auto native_volume_info = volume_desc.info; - native_volume_info.data = {}; - Libs::Graphics::Image native_array(m_runtime_context, scheduler, native_array_info); - Libs::Graphics::Image native_volume(m_runtime_context, scheduler, native_volume_info); - native_volume.CopyImage(native_array); - native_array.CopyImage(native_volume); - Require(name, "Image-owned 2D-array/3D copy state", - native_volume.backing.state.layout == vk::ImageLayout::eTransferSrcOptimal && - native_volume.backing.state.access_mask == - vk::AccessFlagBits2::eTransferRead && - native_array.backing.state.layout == vk::ImageLayout::eGeneral && - native_array.backing.state.access_mask == - (vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead), - "Image::CopyImage did not retain pinned " - "source/destination states"); + auto native_array_info = array_desc.info; + native_array_info.data = {}; + auto native_volume_info = volume_desc.info; + native_volume_info.data = {}; + Libs::Graphics::Image native_array(m_runtime_context, scheduler, + native_array_info); + Libs::Graphics::Image native_volume(m_runtime_context, scheduler, + native_volume_info); + native_volume.CopyImage(native_array); + native_array.CopyImage(native_volume); + Require(name, "Image-owned 2D-array/3D copy state", + native_volume.backing.state.layout == + vk::ImageLayout::eTransferSrcOptimal && + native_volume.backing.state.access_mask == + vk::AccessFlagBits2::eTransferRead && + native_array.backing.state.layout == + vk::ImageLayout::eGeneral && + native_array.backing.state.access_mask == + (vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eTransferRead), + "Image::CopyImage did not retain pinned " + "source/destination states"); - constexpr uint64_t block_alias_offset = 0x23000; - constexpr std::array block_alias_data {0x01234567u, 0x89abcdefu, - 0xfedcba98u, 0x76543210u}; - std::memcpy(memory + block_alias_offset, block_alias_data.data(), - sizeof(block_alias_data)); - auto uncompressed_block = MakeLinearDesc( - base + block_alias_offset, sizeof(block_alias_data), vk::Format::eR32G32B32A32Uint, - Prospero::BufferFormat::k32_32_32_32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 16, 1); - const auto uncompressed_block_image = texture_cache.FindImage(uncompressed_block); - (void)texture_cache.FindTexture(uncompressed_block_image, uncompressed_block); - texture_cache.MarkGpuWritten(uncompressed_block_image); - auto compressed_block = MakeLinearDesc( - base + block_alias_offset, sizeof(block_alias_data), vk::Format::eBc3UnormBlock, - Prospero::BufferFormat::kBc3UNorm, - Prospero::ImageType::kColor2D, {4, 4, 1}, 1, 16, 1); - const auto compressed_block_image = texture_cache.FindImage(compressed_block); - const bool compressed_block_download = - TextureCacheTestAccess::TryDownload(texture_cache, compressed_block_image); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - std::array block_alias_after {}; - std::memcpy(block_alias_after.data(), memory + block_alias_offset, - sizeof(block_alias_after)); - Require(name, "compressed view expansion", - compressed_block_image && compressed_block_image != uncompressed_block_image && - texture_cache.GetImage(compressed_block_image).backing.format == - vk::Format::eBc3UnormBlock && - compressed_block_download && block_alias_after == block_alias_data, - "size-compatible compressed alias did not preserve its native " - "contents"); + constexpr uint64_t block_alias_offset = 0x23000; + constexpr std::array block_alias_data{ + 0x01234567u, 0x89abcdefu, 0xfedcba98u, 0x76543210u}; + std::memcpy(memory + block_alias_offset, block_alias_data.data(), + sizeof(block_alias_data)); + auto uncompressed_block = + MakeLinearDesc(base + block_alias_offset, sizeof(block_alias_data), + vk::Format::eR32G32B32A32Uint, + Prospero::BufferFormat::k32_32_32_32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 16, 1); + const auto uncompressed_block_image = + texture_cache.FindImage(uncompressed_block); + (void)texture_cache.FindTexture(uncompressed_block_image, + uncompressed_block); + texture_cache.MarkGpuWritten(uncompressed_block_image); + auto compressed_block = MakeLinearDesc( + base + block_alias_offset, sizeof(block_alias_data), + vk::Format::eBc3UnormBlock, Prospero::BufferFormat::kBc3UNorm, + Prospero::ImageType::kColor2D, {4, 4, 1}, 1, 16, 1); + const auto compressed_block_image = + texture_cache.FindImage(compressed_block); + const bool compressed_block_download = + TextureCacheTestAccess::TryDownload(texture_cache, + compressed_block_image); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::array block_alias_after{}; + std::memcpy(block_alias_after.data(), memory + block_alias_offset, + sizeof(block_alias_after)); + Require( + name, "compressed view expansion", + compressed_block_image && + compressed_block_image != uncompressed_block_image && + texture_cache.GetImage(compressed_block_image).backing.format == + vk::Format::eBc3UnormBlock && + compressed_block_download && + block_alias_after == block_alias_data, + "size-compatible compressed alias did not preserve its native " + "contents"); - ImageInfo resolve_source_info {}; - resolve_source_info.pixel_format = vk::Format::eR8G8B8A8Unorm; - resolve_source_info.guest_format = - Prospero::BufferFormat::k8_8_8_8UNorm; - resolve_source_info.type = Prospero::ImageType::kColor2D; - resolve_source_info.extent = {4, 4, 1}; - resolve_source_info.resources = {1, 1}; - resolve_source_info.pitch = 4; - resolve_source_info.bytes_per_block = 4; - resolve_source_info.samples = 2; - resolve_source_info.tile_mode = - Prospero::TileMode::kRenderTarget; - ImageInfo resolve_destination_info = resolve_source_info; - resolve_destination_info.type = Prospero::ImageType::kColor2D; - resolve_destination_info.samples = 1; - Libs::Graphics::Image resolve_source(m_runtime_context, scheduler, resolve_source_info); - Libs::Graphics::Image resolve_destination(m_runtime_context, scheduler, - resolve_destination_info); - resolve_destination.Resolve(resolve_source, {}, {}); - Require(name, "Image-owned MSAA resolve", - resolve_source.backing.state.layout == vk::ImageLayout::eTransferSrcOptimal && - resolve_destination.backing.state.layout == - vk::ImageLayout::eTransferDstOptimal, - "Image::Resolve did not issue the MSAA-to-single-sample transfer"); - Libs::Graphics::Image copy_destination(m_runtime_context, scheduler, - resolve_destination_info); - copy_destination.Resolve(resolve_destination, {}, {}); - Require( - name, "Image-owned single-sample resolve copy", - resolve_destination.backing.state.layout == vk::ImageLayout::eTransferSrcOptimal && - copy_destination.backing.state.layout == vk::ImageLayout::eTransferDstOptimal, - "Image::Resolve did not retain the single-sample copy path"); + ImageInfo resolve_source_info{}; + resolve_source_info.pixel_format = vk::Format::eR8G8B8A8Unorm; + resolve_source_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; + resolve_source_info.type = Prospero::ImageType::kColor2D; + resolve_source_info.extent = {4, 4, 1}; + resolve_source_info.resources = {1, 1}; + resolve_source_info.pitch = 4; + resolve_source_info.bytes_per_block = 4; + resolve_source_info.samples = 2; + resolve_source_info.tile_mode = Prospero::TileMode::kRenderTarget; + ImageInfo resolve_destination_info = resolve_source_info; + resolve_destination_info.type = Prospero::ImageType::kColor2D; + resolve_destination_info.samples = 1; + Libs::Graphics::Image resolve_source(m_runtime_context, scheduler, + resolve_source_info); + Libs::Graphics::Image resolve_destination(m_runtime_context, scheduler, + resolve_destination_info); + resolve_destination.Resolve(resolve_source, {}, {}); + Require( + name, "Image-owned MSAA resolve", + resolve_source.backing.state.layout == + vk::ImageLayout::eTransferSrcOptimal && + resolve_destination.backing.state.layout == + vk::ImageLayout::eTransferDstOptimal, + "Image::Resolve did not issue the MSAA-to-single-sample transfer"); + Libs::Graphics::Image copy_destination(m_runtime_context, scheduler, + resolve_destination_info); + copy_destination.Resolve(resolve_destination, {}, {}); + Require(name, "Image-owned single-sample resolve copy", + resolve_destination.backing.state.layout == + vk::ImageLayout::eTransferSrcOptimal && + copy_destination.backing.state.layout == + vk::ImageLayout::eTransferDstOptimal, + "Image::Resolve did not retain the single-sample copy path"); - const std::array multisample_source {0x4000u, 0xc000u}; - std::memcpy(memory + 0x2000, &multisample_source, sizeof(multisample_source)); - auto color_desc = - MakeLinearDesc(base + 0x2000, sizeof(multisample_source), vk::Format::eR16G16Unorm, - Prospero::BufferFormat::k16_16UNorm, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto color_image = texture_cache.FindImage(color_desc); - (void)texture_cache.FindTexture(color_image, color_desc); - texture_cache.MarkGpuWritten(color_image); - const std::array refreshed_multisample_source {0x2000u, 0xe000u}; - Require(name, "unequal-sample source CPU write", - resources.HandleFault(PageFaultAccess::Write, color_desc.info.data.address), - "color-to-MS-depth source did not accept a CPU update"); - std::memcpy(memory + 0x2000, refreshed_multisample_source.data(), - sizeof(refreshed_multisample_source)); - constexpr uint64_t ms_stencil_offset = 0x80000; - constexpr uint64_t ms_stencil_size = 0x10000; - auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer( - command, base + ms_stencil_offset, ms_stencil_size, false, true); - Require(name, "MS stencil buffer allocation", ms_stencil_owner.owner != nullptr, - "failed to create an unequal-sample stencil source"); - command.RetainResourceUntilFence(ms_stencil_owner.owner); - resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, ms_stencil_size, - 0x41414141u); - auto ms_depth_desc = color_desc; - ms_depth_desc.type = BindingType::DepthTarget; - ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size}; - ms_depth_desc.info.pixel_format = vk::Format::eD24UnormS8Uint; - ms_depth_desc.info.guest_format = - Prospero::BufferFormat::k16UNorm; - ms_depth_desc.info.bytes_per_block = 2; - ms_depth_desc.info.samples = 2; - ms_depth_desc.info.type = Prospero::ImageType::kColor2D; - ms_depth_desc.view_info.format = vk::Format::eD24UnormS8Uint; - ms_depth_desc.view_info.aspect = - vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; - ms_depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto ms_depth_image = texture_cache.FindImage(ms_depth_desc); - constexpr uint64_t ms_htile_address = base + 0x2600000; - constexpr uint64_t ms_htile_size = 0x10000; - ms_depth_desc.info.metadata.range = {ms_htile_address, ms_htile_size}; - ms_depth_desc.info.metadata.kind = ImageMetadataKind::Htile; - const auto ms_depth_with_htile = texture_cache.FindImage(ms_depth_desc); - const auto ms_depth_view = - texture_cache.FindDepthTarget(ms_depth_with_htile, ms_depth_desc); - auto sampled_ms_depth = ms_depth_desc; - sampled_ms_depth.type = BindingType::Texture; - sampled_ms_depth.info.metadata = {}; - sampled_ms_depth.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto sampled_ms_depth_image = texture_cache.FindImage(sampled_ms_depth); - Require(name, "unequal-sample depth overlap", - ms_depth_image && ms_depth_with_htile == ms_depth_image && - sampled_ms_depth_image == ms_depth_image && ms_depth_image != color_image && - ms_depth_view != nullptr && - texture_cache.GetImage(ms_depth_image).backing.samples == 2 && - texture_cache.GetImage(ms_depth_image).backing.state.layout == - vk::ImageLayout::eDepthStencilAttachmentOptimal && - texture_cache.GetImage(ms_depth_image).IsGpuModified() && - texture_cache.GetImage(ms_depth_image).info.htile_clear_mask == 0 && - texture_cache.GetImage(ms_depth_image).info.metadata.range == - ms_depth_desc.info.metadata.range && - texture_cache.IsMeta(ms_htile_address) && - !texture_cache.IsMetaCleared(ms_htile_address, 0) && - !resources.GetBufferCache().HasGpuDirtyBytes(base + ms_stencil_offset, - ms_stencil_size) && - !texture_cache.QueryRegion(base + ms_stencil_offset, ms_stencil_size) - .gpu_image_bytes, - "unequal-sample overlap did not run the color-to-MS-depth pass " - "without manufacturing stencil ownership"); - auto& oversized_ms = texture_cache.GetImage(ms_depth_image); - const auto ms_data_size = oversized_ms.info.data.size; - oversized_ms.info.data.size = (32ull << 20) + 4; - const bool oversized_ms_readback = - !TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image); - oversized_ms.info.data.size = ms_data_size; - Require(name, "oversized multisample download rejection", - oversized_ms_readback && oversized_ms.IsGpuModified() && - !oversized_ms.IsBufferModified() && - !resources.GetBufferCache().HasGpuDirtyBytes( - ms_depth_desc.info.data.address, ms_data_size), - "an unsupported multisample source reserved download storage or " - "published Buffer ownership"); + const std::array multisample_source{0x4000u, 0xc000u}; + std::memcpy(memory + 0x2000, &multisample_source, + sizeof(multisample_source)); + auto color_desc = MakeLinearDesc( + base + 0x2000, sizeof(multisample_source), vk::Format::eR16G16Unorm, + Prospero::BufferFormat::k16_16UNorm, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + const auto color_image = texture_cache.FindImage(color_desc); + (void)texture_cache.FindTexture(color_image, color_desc); + texture_cache.MarkGpuWritten(color_image); + const std::array refreshed_multisample_source{0x2000u, + 0xe000u}; + Require(name, "unequal-sample source CPU write", + resources.HandleFault(PageFaultAccess::Write, + color_desc.info.data.address), + "color-to-MS-depth source did not accept a CPU update"); + std::memcpy(memory + 0x2000, refreshed_multisample_source.data(), + sizeof(refreshed_multisample_source)); + constexpr uint64_t ms_stencil_offset = 0x80000; + constexpr uint64_t ms_stencil_size = 0x10000; + auto ms_stencil_owner = resources.GetBufferCache().ObtainBuffer( + command, base + ms_stencil_offset, ms_stencil_size, false, true); + Require(name, "MS stencil buffer allocation", + ms_stencil_owner.owner != nullptr, + "failed to create an unequal-sample stencil source"); + command.RetainResourceUntilFence(ms_stencil_owner.owner); + resources.GetBufferCache().FillBuffer(base + ms_stencil_offset, + ms_stencil_size, 0x41414141u); + auto ms_depth_desc = color_desc; + ms_depth_desc.type = BindingType::DepthTarget; + ms_depth_desc.info.stencil = {base + ms_stencil_offset, ms_stencil_size}; + ms_depth_desc.info.pixel_format = vk::Format::eD24UnormS8Uint; + ms_depth_desc.info.guest_format = Prospero::BufferFormat::k16UNorm; + ms_depth_desc.info.bytes_per_block = 2; + ms_depth_desc.info.samples = 2; + ms_depth_desc.info.type = Prospero::ImageType::kColor2D; + ms_depth_desc.view_info.format = vk::Format::eD24UnormS8Uint; + ms_depth_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + ms_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto ms_depth_image = texture_cache.FindImage(ms_depth_desc); + constexpr uint64_t ms_htile_address = base + 0x2600000; + constexpr uint64_t ms_htile_size = 0x10000; + ms_depth_desc.info.metadata.range = {ms_htile_address, ms_htile_size}; + ms_depth_desc.info.metadata.kind = ImageMetadataKind::Htile; + const auto ms_depth_with_htile = texture_cache.FindImage(ms_depth_desc); + const auto ms_depth_view = + texture_cache.FindDepthTarget(ms_depth_with_htile, ms_depth_desc); + auto sampled_ms_depth = ms_depth_desc; + sampled_ms_depth.type = BindingType::Texture; + sampled_ms_depth.info.metadata = {}; + sampled_ms_depth.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto sampled_ms_depth_image = + texture_cache.FindImage(sampled_ms_depth); + Require( + name, "unequal-sample depth overlap", + ms_depth_image && ms_depth_with_htile == ms_depth_image && + sampled_ms_depth_image == ms_depth_image && + ms_depth_image != color_image && ms_depth_view != nullptr && + texture_cache.GetImage(ms_depth_image).backing.samples == 2 && + texture_cache.GetImage(ms_depth_image).backing.state.layout == + vk::ImageLayout::eDepthStencilAttachmentOptimal && + texture_cache.GetImage(ms_depth_image).IsGpuModified() && + texture_cache.GetImage(ms_depth_image).info.htile_clear_mask == + 0 && + texture_cache.GetImage(ms_depth_image).info.metadata.range == + ms_depth_desc.info.metadata.range && + texture_cache.IsMeta(ms_htile_address) && + !texture_cache.IsMetaCleared(ms_htile_address, 0) && + !resources.GetBufferCache().HasGpuDirtyBytes( + base + ms_stencil_offset, ms_stencil_size) && + !texture_cache + .QueryRegion(base + ms_stencil_offset, ms_stencil_size) + .gpu_image_bytes, + "unequal-sample overlap did not run the color-to-MS-depth pass " + "without manufacturing stencil ownership"); + auto &oversized_ms = texture_cache.GetImage(ms_depth_image); + const auto ms_data_size = oversized_ms.info.data.size; + oversized_ms.info.data.size = (32ull << 20) + 4; + const bool oversized_ms_readback = + !TextureCacheTestAccess::TryDownload(texture_cache, ms_depth_image); + oversized_ms.info.data.size = ms_data_size; + Require(name, "oversized multisample download rejection", + oversized_ms_readback && oversized_ms.IsGpuModified() && + !oversized_ms.IsBufferModified() && + !resources.GetBufferCache().HasGpuDirtyBytes( + ms_depth_desc.info.data.address, ms_data_size), + "an unsupported multisample source reserved download storage or " + "published Buffer ownership"); - const std::array multisample_source_4x {0x0000u, 0x4000u, 0x8000u, - 0xffffu}; - std::memcpy(memory + 0x7000, multisample_source_4x.data(), - sizeof(multisample_source_4x)); - auto color_desc_4x = MakeLinearDesc( - base + 0x7000, sizeof(multisample_source_4x), vk::Format::eR16G16B16A16Unorm, - Prospero::BufferFormat::k16_16_16_16UNorm, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1); - const auto color_image_4x = texture_cache.FindImage(color_desc_4x); - auto ms_depth_desc_4x = color_desc_4x; - ms_depth_desc_4x.type = BindingType::DepthTarget; - ms_depth_desc_4x.info.pixel_format = vk::Format::eD16Unorm; - ms_depth_desc_4x.info.guest_format = - Prospero::BufferFormat::k16UNorm; - ms_depth_desc_4x.info.bytes_per_block = 2; - ms_depth_desc_4x.info.samples = 4; - ms_depth_desc_4x.info.type = Prospero::ImageType::kColor2D; - ms_depth_desc_4x.view_info.format = vk::Format::eD16Unorm; - ms_depth_desc_4x.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - ms_depth_desc_4x.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto ms_depth_image_4x = texture_cache.FindImage(ms_depth_desc_4x); - Require(name, "four-sample depth overlap", - ms_depth_image_4x && ms_depth_image_4x != color_image_4x && - texture_cache.GetImage(ms_depth_image_4x).backing.samples == 4, - "four-sample color packing did not produce a depth image"); + const std::array multisample_source_4x{0x0000u, 0x4000u, + 0x8000u, 0xffffu}; + std::memcpy(memory + 0x7000, multisample_source_4x.data(), + sizeof(multisample_source_4x)); + auto color_desc_4x = + MakeLinearDesc(base + 0x7000, sizeof(multisample_source_4x), + vk::Format::eR16G16B16A16Unorm, + Prospero::BufferFormat::k16_16_16_16UNorm, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1); + const auto color_image_4x = texture_cache.FindImage(color_desc_4x); + auto ms_depth_desc_4x = color_desc_4x; + ms_depth_desc_4x.type = BindingType::DepthTarget; + ms_depth_desc_4x.info.pixel_format = vk::Format::eD16Unorm; + ms_depth_desc_4x.info.guest_format = Prospero::BufferFormat::k16UNorm; + ms_depth_desc_4x.info.bytes_per_block = 2; + ms_depth_desc_4x.info.samples = 4; + ms_depth_desc_4x.info.type = Prospero::ImageType::kColor2D; + ms_depth_desc_4x.view_info.format = vk::Format::eD16Unorm; + ms_depth_desc_4x.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + ms_depth_desc_4x.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto ms_depth_image_4x = texture_cache.FindImage(ms_depth_desc_4x); + Require(name, "four-sample depth overlap", + ms_depth_image_4x && ms_depth_image_4x != color_image_4x && + texture_cache.GetImage(ms_depth_image_4x).backing.samples == + 4, + "four-sample color packing did not produce a depth image"); - const auto ms_depth_backing = texture_cache.GetImage(ms_depth_image).backing.image; - const auto first_stencil_association = - texture_cache.FindImageFromRange(base + ms_stencil_offset, ms_stencil_size, false); - Require( - name, "stencil association shape", - first_stencil_association && first_stencil_association != ms_depth_image && - texture_cache.GetImage(first_stencil_association).info.pixel_format == - vk::Format::eUndefined && - texture_cache.GetImage(first_stencil_association).backing.image == nullptr && - texture_cache.GetImage(first_stencil_association).depth_id == ms_depth_image, - "stencil address did not create a lightweight depth association"); - Require(name, "stencil association download rejection", - !TextureCacheTestAccess::TryDownload(texture_cache, first_stencil_association), - "a lightweight stencil association entered image download planning"); + const auto ms_depth_backing = + texture_cache.GetImage(ms_depth_image).backing.image; + const auto first_stencil_association = texture_cache.FindImageFromRange( + base + ms_stencil_offset, ms_stencil_size, false); + Require( + name, "stencil association shape", + first_stencil_association && + first_stencil_association != ms_depth_image && + texture_cache.GetImage(first_stencil_association) + .info.pixel_format == vk::Format::eUndefined && + texture_cache.GetImage(first_stencil_association).backing.image == + nullptr && + texture_cache.GetImage(first_stencil_association).depth_id == + ms_depth_image, + "stencil address did not create a lightweight depth association"); + Require( + name, "stencil association download rejection", + !TextureCacheTestAccess::TryDownload(texture_cache, + first_stencil_association), + "a lightweight stencil association entered image download planning"); - auto exact_ms_depth_alias = ms_depth_desc; - exact_ms_depth_alias.type = BindingType::Texture; - exact_ms_depth_alias.info.stencil = {}; - exact_ms_depth_alias.info.pixel_format = vk::Format::eR8G8B8A8Unorm; - exact_ms_depth_alias.info.guest_format = - Prospero::BufferFormat::k8_8_8_8UNorm; - exact_ms_depth_alias.info.bytes_per_block = 4; - exact_ms_depth_alias.view_info.format = vk::Format::eR8G8B8A8Unorm; - exact_ms_depth_alias.view_info.aspect = vk::ImageAspectFlagBits::eColor; - exact_ms_depth_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto exact_ms_depth_image = texture_cache.FindImage(exact_ms_depth_alias, true); - Require(name, "exact multisample depth/stencil coexistence", - exact_ms_depth_image && exact_ms_depth_image != ms_depth_image && - TextureCacheTestAccess::Contains(texture_cache, ms_depth_image) && - texture_cache.GetImage(ms_depth_image).backing.image == ms_depth_backing && - texture_cache.GetImage(first_stencil_association).depth_id == - ms_depth_image, - "exact multisample depth/stencil lookup replaced the old image " - "or stole its stencil association"); + auto exact_ms_depth_alias = ms_depth_desc; + exact_ms_depth_alias.type = BindingType::Texture; + exact_ms_depth_alias.info.stencil = {}; + exact_ms_depth_alias.info.pixel_format = vk::Format::eR8G8B8A8Unorm; + exact_ms_depth_alias.info.guest_format = + Prospero::BufferFormat::k8_8_8_8UNorm; + exact_ms_depth_alias.info.bytes_per_block = 4; + exact_ms_depth_alias.view_info.format = vk::Format::eR8G8B8A8Unorm; + exact_ms_depth_alias.view_info.aspect = vk::ImageAspectFlagBits::eColor; + exact_ms_depth_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto exact_ms_depth_image = + texture_cache.FindImage(exact_ms_depth_alias, true); + Require( + name, "exact multisample depth/stencil coexistence", + exact_ms_depth_image && exact_ms_depth_image != ms_depth_image && + TextureCacheTestAccess::Contains(texture_cache, ms_depth_image) && + texture_cache.GetImage(ms_depth_image).backing.image == + ms_depth_backing && + texture_cache.GetImage(first_stencil_association).depth_id == + ms_depth_image, + "exact multisample depth/stencil lookup replaced the old image " + "or stole its stencil association"); - constexpr uint64_t second_stencil_offset = 0x90000; - auto switched_ms_depth = ms_depth_desc; - switched_ms_depth.info.stencil = {base + second_stencil_offset, ms_stencil_size}; - const auto switched_ms_depth_id = texture_cache.FindImage(switched_ms_depth); - TextureCacheTestAccess::AssociateStencil(texture_cache, switched_ms_depth_id, - switched_ms_depth.info.stencil); - const auto second_stencil_association = texture_cache.FindImageFromRange( - base + second_stencil_offset, ms_stencil_size, false); - Require( - name, "stencil association switch", - switched_ms_depth_id == ms_depth_image && - texture_cache.GetImage(switched_ms_depth_id).backing.image == - ms_depth_backing && - texture_cache.FindImageFromRange(base + ms_stencil_offset, ms_stencil_size, - false) == first_stencil_association && - texture_cache.GetImage(first_stencil_association).depth_id == ms_depth_image && - second_stencil_association && - texture_cache.GetImage(second_stencil_association).depth_id == ms_depth_image, - "changing stencil address recreated depth or discarded an " - "existing lightweight association"); + constexpr uint64_t second_stencil_offset = 0x90000; + auto switched_ms_depth = ms_depth_desc; + switched_ms_depth.info.stencil = {base + second_stencil_offset, + ms_stencil_size}; + const auto switched_ms_depth_id = + texture_cache.FindImage(switched_ms_depth); + TextureCacheTestAccess::AssociateStencil( + texture_cache, switched_ms_depth_id, switched_ms_depth.info.stencil); + const auto second_stencil_association = texture_cache.FindImageFromRange( + base + second_stencil_offset, ms_stencil_size, false); + Require(name, "stencil association switch", + switched_ms_depth_id == ms_depth_image && + texture_cache.GetImage(switched_ms_depth_id).backing.image == + ms_depth_backing && + texture_cache.FindImageFromRange(base + ms_stencil_offset, + ms_stencil_size, false) == + first_stencil_association && + texture_cache.GetImage(first_stencil_association).depth_id == + ms_depth_image && + second_stencil_association && + texture_cache.GetImage(second_stencil_association).depth_id == + ms_depth_image, + "changing stencil address recreated depth or discarded an " + "existing lightweight association"); - constexpr uint64_t added_stencil_depth_offset = 0x60000; - constexpr uint64_t added_stencil_offset = 0x70000; - constexpr uint64_t added_stencil_size = 0x10000; - constexpr float added_stencil_depth_value = 0.75f; - std::memcpy(memory + added_stencil_depth_offset, &added_stencil_depth_value, - sizeof(added_stencil_depth_value)); - auto depth_only_source = - MakeLinearDesc(base + added_stencil_depth_offset, 0x10000, vk::Format::eD32Sfloat, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - depth_only_source.type = BindingType::DepthTarget; - depth_only_source.view_info.format = vk::Format::eD32Sfloat; - depth_only_source.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - depth_only_source.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto depth_only_source_image = texture_cache.FindImage(depth_only_source); + constexpr uint64_t added_stencil_depth_offset = 0x60000; + constexpr uint64_t added_stencil_offset = 0x70000; + constexpr uint64_t added_stencil_size = 0x10000; + constexpr float added_stencil_depth_value = 0.75f; + std::memcpy(memory + added_stencil_depth_offset, + &added_stencil_depth_value, + sizeof(added_stencil_depth_value)); + auto depth_only_source = MakeLinearDesc( + base + added_stencil_depth_offset, 0x10000, vk::Format::eD32Sfloat, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + depth_only_source.type = BindingType::DepthTarget; + depth_only_source.view_info.format = vk::Format::eD32Sfloat; + depth_only_source.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + depth_only_source.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto depth_only_source_image = + texture_cache.FindImage(depth_only_source); - auto combined_destination = depth_only_source; - combined_destination.info.stencil = {base + added_stencil_offset, added_stencil_size}; - combined_destination.info.pixel_format = vk::Format::eD32SfloatS8Uint; - combined_destination.view_info.format = vk::Format::eD32SfloatS8Uint; - combined_destination.view_info.aspect = - vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; - const auto combined_destination_image = texture_cache.FindImage(combined_destination); - TextureCacheTestAccess::AssociateStencil(texture_cache, combined_destination_image, - combined_destination.info.stencil); - const auto combined_stencil_association = texture_cache.FindImageFromRange( - base + added_stencil_offset, added_stencil_size, false); - Require(name, "depth-only to combined association", - combined_destination_image != depth_only_source_image && - combined_stencil_association && - texture_cache.GetImage(combined_stencil_association).depth_id == - combined_destination_image && - texture_cache.GetImage(combined_stencil_association).backing.image == - nullptr, - "stencil addition did not recreate depth and register its " - "lightweight association"); + auto combined_destination = depth_only_source; + combined_destination.info.stencil = {base + added_stencil_offset, + added_stencil_size}; + combined_destination.info.pixel_format = vk::Format::eD32SfloatS8Uint; + combined_destination.view_info.format = vk::Format::eD32SfloatS8Uint; + combined_destination.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + const auto combined_destination_image = + texture_cache.FindImage(combined_destination); + TextureCacheTestAccess::AssociateStencil( + texture_cache, combined_destination_image, + combined_destination.info.stencil); + const auto combined_stencil_association = + texture_cache.FindImageFromRange(base + added_stencil_offset, + added_stencil_size, false); + Require( + name, "depth-only to combined association", + combined_destination_image != depth_only_source_image && + combined_stencil_association && + texture_cache.GetImage(combined_stencil_association).depth_id == + combined_destination_image && + texture_cache.GetImage(combined_stencil_association) + .backing.image == nullptr, + "stencil addition did not recreate depth and register its " + "lightweight association"); - const uint32_t mirror_value = 0xa1b2c3d4u; - std::memcpy(memory + 0x5000, &mirror_value, sizeof(mirror_value)); - auto mirror_desc = - MakeLinearDesc(base + 0x5000, sizeof(mirror_value), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto mirror_image = texture_cache.FindImage(mirror_desc); - (void)texture_cache.FindTexture(mirror_image, mirror_desc); - texture_cache.MarkGpuWritten(mirror_image); - constexpr uint32_t mirror_cpu_value = 0x0f1e2d3cu; - Require(name, "CPU-dirty formatted mirror fault", - resources.HandleFault(PageFaultAccess::Write, mirror_desc.info.data.address), - "GPU image did not accept a CPU write before Buffer mirroring"); - std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value)); - auto mirror_binding = resources.GetBufferCache().ObtainBuffer( - command, base + 0x5000, sizeof(mirror_value), false, true, true); - Require(name, "CPU-dirty formatted mirror source", - mirror_binding.buffer != nullptr && - !texture_cache.GetImage(mirror_image).IsBufferModified(), - "formatted mirror did not expose a readable Buffer source"); - if (mirror_binding.owner != nullptr) { - command.RetainResourceUntilFence(mirror_binding.owner); - } - auto mirror_cpu_readback = - CreateHostBuffer(name, sizeof(mirror_cpu_value), - vk::BufferUsageFlagBits::eTransferDst, std::vector {0}); - const vk::BufferCopy mirror_cpu_copy {mirror_binding.offset, 0, - sizeof(mirror_cpu_value)}; - command.Handle().copyBuffer(mirror_binding.buffer, mirror_cpu_readback.buffer, 1, - &mirror_cpu_copy); - HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - auto mirror_refresh_desc = mirror_desc; - const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc); - (void)texture_cache.FindTexture(mirror_refresh, mirror_refresh_desc); - Require(name, "buffer-to-image ownership", - mirror_refresh == mirror_image && - !texture_cache.GetImage(mirror_refresh).IsBufferModified(), - "buffer-backed refresh incorrectly transferred dirty ownership " - "to the image"); - texture_cache.MarkGpuWritten(mirror_refresh); + const uint32_t mirror_value = 0xa1b2c3d4u; + std::memcpy(memory + 0x5000, &mirror_value, sizeof(mirror_value)); + auto mirror_desc = + MakeLinearDesc(base + 0x5000, sizeof(mirror_value), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto mirror_image = texture_cache.FindImage(mirror_desc); + (void)texture_cache.FindTexture(mirror_image, mirror_desc); + texture_cache.MarkGpuWritten(mirror_image); + constexpr uint32_t mirror_cpu_value = 0x0f1e2d3cu; + Require(name, "CPU-dirty formatted mirror fault", + resources.HandleFault(PageFaultAccess::Write, + mirror_desc.info.data.address), + "GPU image did not accept a CPU write before Buffer mirroring"); + std::memcpy(memory + 0x5000, &mirror_cpu_value, sizeof(mirror_cpu_value)); + auto mirror_binding = resources.GetBufferCache().ObtainBuffer( + command, base + 0x5000, sizeof(mirror_value), false, true, true); + Require(name, "CPU-dirty formatted mirror source", + mirror_binding.buffer != nullptr && + !texture_cache.GetImage(mirror_image).IsBufferModified(), + "formatted mirror did not expose a readable Buffer source"); + if (mirror_binding.owner != nullptr) { + command.RetainResourceUntilFence(mirror_binding.owner); + } + auto mirror_cpu_readback = CreateHostBuffer( + name, sizeof(mirror_cpu_value), vk::BufferUsageFlagBits::eTransferDst, + std::vector{0}); + const vk::BufferCopy mirror_cpu_copy{mirror_binding.offset, 0, + sizeof(mirror_cpu_value)}; + command.Handle().copyBuffer(mirror_binding.buffer, + mirror_cpu_readback.buffer, 1, + &mirror_cpu_copy); + HostReadBarrier(mirror_cpu_readback.buffer, mirror_cpu_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + auto mirror_refresh_desc = mirror_desc; + const auto mirror_refresh = texture_cache.FindImage(mirror_refresh_desc); + (void)texture_cache.FindTexture(mirror_refresh, mirror_refresh_desc); + Require(name, "buffer-to-image ownership", + mirror_refresh == mirror_image && + !texture_cache.GetImage(mirror_refresh).IsBufferModified(), + "buffer-backed refresh incorrectly transferred dirty ownership " + "to the image"); + texture_cache.MarkGpuWritten(mirror_refresh); - constexpr uint64_t exact_buffer_offset = 0x90000; - constexpr uint32_t exact_buffer_value = 0x3f234567u; - std::memcpy(memory + exact_buffer_offset, &exact_buffer_value, - sizeof(exact_buffer_value)); - auto exact_buffer_desc = MakeLinearDesc( - base + exact_buffer_offset, sizeof(exact_buffer_value), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto exact_buffer_image = texture_cache.FindImage(exact_buffer_desc); - (void)texture_cache.FindTexture(exact_buffer_image, exact_buffer_desc); - texture_cache.MarkGpuWritten(exact_buffer_image); - auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer( - command, exact_buffer_desc.info.data.address, exact_buffer_desc.info.data.size, - false, true, true); - Require(name, "exact replacement Buffer synchronization", - exact_buffer_binding.buffer != nullptr && - !texture_cache.GetImage(exact_buffer_image).IsBufferModified() && - texture_cache.GetImage(exact_buffer_image).IsGpuModified(), - "exact replacement copy transferred cache ownership"); - if (exact_buffer_binding.owner != nullptr) { - command.RetainResourceUntilFence(exact_buffer_binding.owner); - } - auto exact_float_desc = exact_buffer_desc; - exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat; - exact_float_desc.info.guest_format = - Prospero::BufferFormat::k32Float; - exact_float_desc.view_info.format = vk::Format::eR32Sfloat; - const auto exact_float_image = texture_cache.FindImage(exact_float_desc, true); - (void)texture_cache.FindTexture(exact_float_image, exact_float_desc); - Require(name, "Buffer-superseded exact coexistence", - exact_float_image != exact_buffer_image && - TextureCacheTestAccess::Contains(texture_cache, exact_buffer_image) && - texture_cache.GetImage(exact_buffer_image).IsGpuModified() && - !texture_cache.GetImage(exact_float_image).IsBufferModified() && - !texture_cache.GetImage(exact_float_image).IsGpuModified(), - "exact-format lookup retired its old record or transferred Buffer " - "ownership"); - auto exact_buffer_readback = - CreateHostBuffer(name, sizeof(exact_buffer_value), - vk::BufferUsageFlagBits::eTransferDst, std::vector {0}); - auto& exact_float_native = texture_cache.GetImage(exact_float_image); - exact_float_native.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command.Handle()); - vk::BufferImageCopy exact_float_copy {}; - exact_float_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - exact_float_copy.imageSubresource.layerCount = 1; - exact_float_copy.imageExtent = {1, 1, 1}; - command.Handle().copyImageToBuffer(exact_float_native.backing.image, - vk::ImageLayout::eTransferSrcOptimal, - exact_buffer_readback.buffer, 1, &exact_float_copy); - HostReadBarrier(exact_buffer_readback.buffer, exact_buffer_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); + constexpr uint64_t exact_buffer_offset = 0x90000; + constexpr uint32_t exact_buffer_value = 0x3f234567u; + std::memcpy(memory + exact_buffer_offset, &exact_buffer_value, + sizeof(exact_buffer_value)); + auto exact_buffer_desc = + MakeLinearDesc(base + exact_buffer_offset, sizeof(exact_buffer_value), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto exact_buffer_image = + texture_cache.FindImage(exact_buffer_desc); + (void)texture_cache.FindTexture(exact_buffer_image, exact_buffer_desc); + texture_cache.MarkGpuWritten(exact_buffer_image); + auto exact_buffer_binding = resources.GetBufferCache().ObtainBuffer( + command, exact_buffer_desc.info.data.address, + exact_buffer_desc.info.data.size, false, true, true); + Require( + name, "exact replacement Buffer synchronization", + exact_buffer_binding.buffer != nullptr && + !texture_cache.GetImage(exact_buffer_image).IsBufferModified() && + texture_cache.GetImage(exact_buffer_image).IsGpuModified(), + "exact replacement copy transferred cache ownership"); + if (exact_buffer_binding.owner != nullptr) { + command.RetainResourceUntilFence(exact_buffer_binding.owner); + } + auto exact_float_desc = exact_buffer_desc; + exact_float_desc.info.pixel_format = vk::Format::eR32Sfloat; + exact_float_desc.info.guest_format = Prospero::BufferFormat::k32Float; + exact_float_desc.view_info.format = vk::Format::eR32Sfloat; + const auto exact_float_image = + texture_cache.FindImage(exact_float_desc, true); + (void)texture_cache.FindTexture(exact_float_image, exact_float_desc); + Require( + name, "Buffer-superseded exact coexistence", + exact_float_image != exact_buffer_image && + TextureCacheTestAccess::Contains(texture_cache, + exact_buffer_image) && + texture_cache.GetImage(exact_buffer_image).IsGpuModified() && + !texture_cache.GetImage(exact_float_image).IsBufferModified() && + !texture_cache.GetImage(exact_float_image).IsGpuModified(), + "exact-format lookup retired its old record or transferred Buffer " + "ownership"); + auto exact_buffer_readback = CreateHostBuffer( + name, sizeof(exact_buffer_value), + vk::BufferUsageFlagBits::eTransferDst, std::vector{0}); + auto &exact_float_native = texture_cache.GetImage(exact_float_image); + exact_float_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + vk::BufferImageCopy exact_float_copy{}; + exact_float_copy.imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eColor; + exact_float_copy.imageSubresource.layerCount = 1; + exact_float_copy.imageExtent = {1, 1, 1}; + command.Handle().copyImageToBuffer(exact_float_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, + exact_buffer_readback.buffer, 1, + &exact_float_copy); + HostReadBarrier(exact_buffer_readback.buffer, exact_buffer_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); - const uint32_t clear_source = 0x01010101u; - std::memcpy(memory + 0x6000, &clear_source, sizeof(clear_source)); - auto clear_desc = - MakeLinearDesc(base + 0x6000, sizeof(clear_source), vk::Format::eR8G8B8A8Unorm, - Prospero::BufferFormat::k8_8_8_8UNorm, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto clear_image = texture_cache.FindImage(clear_desc); - auto buffer_write = resources.GetBufferCache().ObtainBuffer( - command, base + 0x6000, sizeof(clear_source), true, false, true); - Require(name, "buffer-write ownership", - buffer_write.buffer != nullptr && - texture_cache.GetImage(clear_image).IsBufferModified(), - "formatted buffer write did not supersede the image"); - if (buffer_write.owner != nullptr) { - command.RetainResourceUntilFence(buffer_write.owner); - } - Require(name, "buffer-owned full image clear", - texture_cache.ClearImageFromBuffer(command, base + 0x6000, sizeof(clear_source), - 0xaabbccddu) && - !texture_cache.GetImage(clear_image).IsBufferModified() && - texture_cache.GetImage(clear_image).IsGpuModified(), - "full image clear retained dual buffer/image GPU ownership"); - texture_cache.MarkGpuWritten(clear_image); - auto clear_read_desc = clear_desc; - Require(name, "post-clear image reuse", - texture_cache.FindImage(clear_read_desc) == clear_image, - "cleared image could not be read or written after ownership " - "transfer"); + const uint32_t clear_source = 0x01010101u; + std::memcpy(memory + 0x6000, &clear_source, sizeof(clear_source)); + auto clear_desc = MakeLinearDesc( + base + 0x6000, sizeof(clear_source), vk::Format::eR8G8B8A8Unorm, + Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + const auto clear_image = texture_cache.FindImage(clear_desc); + auto buffer_write = resources.GetBufferCache().ObtainBuffer( + command, base + 0x6000, sizeof(clear_source), true, false, true); + Require(name, "buffer-write ownership", + buffer_write.buffer != nullptr && + texture_cache.GetImage(clear_image).IsBufferModified(), + "formatted buffer write did not supersede the image"); + if (buffer_write.owner != nullptr) { + command.RetainResourceUntilFence(buffer_write.owner); + } + Require(name, "buffer-owned full image clear", + texture_cache.ClearImageFromBuffer( + command, base + 0x6000, sizeof(clear_source), 0xaabbccddu) && + !texture_cache.GetImage(clear_image).IsBufferModified() && + texture_cache.GetImage(clear_image).IsGpuModified(), + "full image clear retained dual buffer/image GPU ownership"); + texture_cache.MarkGpuWritten(clear_image); + auto clear_read_desc = clear_desc; + Require(name, "post-clear image reuse", + texture_cache.FindImage(clear_read_desc) == clear_image, + "cleared image could not be read or written after ownership " + "transfer"); - constexpr uint64_t partial_image_offset = 0xa000; - constexpr uint64_t partial_buffer_offset = 0xa010; - constexpr uint64_t partial_clean_offset = 0xa020; - constexpr uint32_t partial_image_value = 0x31415926u; - constexpr uint32_t partial_buffer_value = 0x27182818u; - constexpr uint32_t partial_clean_value = 0xabcdef01u; - std::memcpy(memory + partial_clean_offset, &partial_clean_value, - sizeof(partial_clean_value)); - auto partial_write = resources.GetBufferCache().ObtainBuffer( - command, base + partial_image_offset, sizeof(partial_image_value), true, false); - Require(name, "partial-page buffer allocation", partial_write.owner != nullptr, - "partial-page GPU write did not create a native buffer"); - command.RetainResourceUntilFence(partial_write.owner); - resources.GetBufferCache().FillBuffer(base + partial_image_offset, - sizeof(partial_image_value), partial_image_value); - resources.GetBufferCache().FillBuffer( - base + partial_buffer_offset, sizeof(partial_buffer_value), partial_buffer_value); - auto partial_desc = MakeLinearDesc( - base + partial_image_offset, sizeof(partial_image_value), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto partial_image = texture_cache.FindImage(partial_desc); - Require(name, "partial-page image upload", - !texture_cache.GetImage(partial_image).IsGpuModified(), - "image upload incorrectly consumed buffer dirty ownership"); - auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer( - command, base + partial_image_offset, sizeof(partial_image_value), false, true, - true); - Require(name, "partial-page image mirror", - partial_image_mirror.buffer != nullptr && - partial_image_mirror.owner != nullptr && - !texture_cache.GetImage(partial_image).IsGpuModified(), - "same-page image upload lost its buffer source"); - command.RetainResourceUntilFence(partial_image_mirror.owner); - const auto partial_clean_source = resources.GetBufferCache().ObtainBufferForImage( - base + partial_clean_offset, sizeof(partial_clean_value)); - Require(name, "partial-page clean source", partial_clean_source.buffer != nullptr, - "clean same-page bytes inherited unrelated buffer ownership"); - Require(name, "partial-page remaining fault", - resources.HandleFault(PageFaultAccess::Read, base + partial_buffer_offset), - "same-page buffer ownership was lost after image transfer"); - uint32_t partial_buffer_backing = 0; - std::memcpy(&partial_buffer_backing, memory + partial_buffer_offset, - sizeof(partial_buffer_backing)); - uint32_t partial_image_backing = 0; - std::memcpy(&partial_image_backing, memory + partial_image_offset, - sizeof(partial_image_backing)); - Require(name, "partial-page readback values", - partial_image_backing == partial_image_value && - partial_buffer_backing == partial_buffer_value, - "same-page image/buffer readback published incorrect bytes"); - constexpr uint32_t partial_cpu_refresh_value = 0x13579bdfu; - Require(name, "partial-page CPU write fault", - resources.HandleFault(PageFaultAccess::Write, base + partial_clean_offset), - "same-page CPU write did not invalidate cached ownership"); - std::memcpy(memory + partial_clean_offset, &partial_cpu_refresh_value, - sizeof(partial_cpu_refresh_value)); - const auto partial_cpu_refresh_source = resources.GetBufferCache().ObtainBufferForImage( - base + partial_clean_offset, sizeof(partial_cpu_refresh_value)); - Require(name, "partial-page CPU refresh source", - partial_cpu_refresh_source.buffer != nullptr, - "CPU-dirty same-page bytes did not resolve through the cached " - "buffer"); - auto partial_cpu_refresh_readback = - CreateHostBuffer(name, sizeof(partial_cpu_refresh_value), - vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy partial_cpu_refresh_copy {partial_cpu_refresh_source.offset, 0, - sizeof(partial_cpu_refresh_value)}; - command.Handle().copyBuffer(partial_cpu_refresh_source.buffer->Handle(), - partial_cpu_refresh_readback.buffer, 1, - &partial_cpu_refresh_copy); - vk::BufferMemoryBarrier partial_cpu_refresh_barrier {}; - partial_cpu_refresh_barrier.sType = vk::StructureType::eBufferMemoryBarrier; - partial_cpu_refresh_barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - partial_cpu_refresh_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - partial_cpu_refresh_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - partial_cpu_refresh_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - partial_cpu_refresh_barrier.buffer = partial_cpu_refresh_readback.buffer; - partial_cpu_refresh_barrier.size = partial_cpu_refresh_readback.size; - command.Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, - &partial_cpu_refresh_barrier, 0, nullptr); + constexpr uint64_t partial_image_offset = 0xa000; + constexpr uint64_t partial_buffer_offset = 0xa010; + constexpr uint64_t partial_clean_offset = 0xa020; + constexpr uint32_t partial_image_value = 0x31415926u; + constexpr uint32_t partial_buffer_value = 0x27182818u; + constexpr uint32_t partial_clean_value = 0xabcdef01u; + std::memcpy(memory + partial_clean_offset, &partial_clean_value, + sizeof(partial_clean_value)); + auto partial_write = resources.GetBufferCache().ObtainBuffer( + command, base + partial_image_offset, sizeof(partial_image_value), + true, false); + Require(name, "partial-page buffer allocation", + partial_write.owner != nullptr, + "partial-page GPU write did not create a native buffer"); + command.RetainResourceUntilFence(partial_write.owner); + resources.GetBufferCache().FillBuffer(base + partial_image_offset, + sizeof(partial_image_value), + partial_image_value); + resources.GetBufferCache().FillBuffer(base + partial_buffer_offset, + sizeof(partial_buffer_value), + partial_buffer_value); + auto partial_desc = MakeLinearDesc( + base + partial_image_offset, sizeof(partial_image_value), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto partial_image = texture_cache.FindImage(partial_desc); + Require(name, "partial-page image upload", + !texture_cache.GetImage(partial_image).IsGpuModified(), + "image upload incorrectly consumed buffer dirty ownership"); + auto partial_image_mirror = resources.GetBufferCache().ObtainBuffer( + command, base + partial_image_offset, sizeof(partial_image_value), + false, true, true); + Require(name, "partial-page image mirror", + partial_image_mirror.buffer != nullptr && + partial_image_mirror.owner != nullptr && + !texture_cache.GetImage(partial_image).IsGpuModified(), + "same-page image upload lost its buffer source"); + command.RetainResourceUntilFence(partial_image_mirror.owner); + const auto partial_clean_source = + resources.GetBufferCache().ObtainBufferForImage( + base + partial_clean_offset, sizeof(partial_clean_value)); + Require(name, "partial-page clean source", + partial_clean_source.buffer != nullptr, + "clean same-page bytes inherited unrelated buffer ownership"); + Require(name, "partial-page remaining fault", + resources.HandleFault(PageFaultAccess::Read, + base + partial_buffer_offset), + "same-page buffer ownership was lost after image transfer"); + uint32_t partial_buffer_backing = 0; + std::memcpy(&partial_buffer_backing, memory + partial_buffer_offset, + sizeof(partial_buffer_backing)); + uint32_t partial_image_backing = 0; + std::memcpy(&partial_image_backing, memory + partial_image_offset, + sizeof(partial_image_backing)); + Require(name, "partial-page readback values", + partial_image_backing == partial_image_value && + partial_buffer_backing == partial_buffer_value, + "same-page image/buffer readback published incorrect bytes"); + constexpr uint32_t partial_cpu_refresh_value = 0x13579bdfu; + Require(name, "partial-page CPU write fault", + resources.HandleFault(PageFaultAccess::Write, + base + partial_clean_offset), + "same-page CPU write did not invalidate cached ownership"); + std::memcpy(memory + partial_clean_offset, &partial_cpu_refresh_value, + sizeof(partial_cpu_refresh_value)); + const auto partial_cpu_refresh_source = + resources.GetBufferCache().ObtainBufferForImage( + base + partial_clean_offset, sizeof(partial_cpu_refresh_value)); + Require(name, "partial-page CPU refresh source", + partial_cpu_refresh_source.buffer != nullptr, + "CPU-dirty same-page bytes did not resolve through the cached " + "buffer"); + auto partial_cpu_refresh_readback = + CreateHostBuffer(name, sizeof(partial_cpu_refresh_value), + vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy partial_cpu_refresh_copy{ + partial_cpu_refresh_source.offset, 0, + sizeof(partial_cpu_refresh_value)}; + command.Handle().copyBuffer(partial_cpu_refresh_source.buffer->Handle(), + partial_cpu_refresh_readback.buffer, 1, + &partial_cpu_refresh_copy); + vk::BufferMemoryBarrier partial_cpu_refresh_barrier{}; + partial_cpu_refresh_barrier.sType = + vk::StructureType::eBufferMemoryBarrier; + partial_cpu_refresh_barrier.srcAccessMask = + vk::AccessFlagBits::eTransferWrite; + partial_cpu_refresh_barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + partial_cpu_refresh_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + partial_cpu_refresh_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + partial_cpu_refresh_barrier.buffer = partial_cpu_refresh_readback.buffer; + partial_cpu_refresh_barrier.size = partial_cpu_refresh_readback.size; + command.Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, + nullptr, 1, &partial_cpu_refresh_barrier, + 0, nullptr); - const uint32_t fault_a = 0x01020304u; - const uint32_t fault_b = 0x11121314u; - std::memcpy(memory + 0x8000, &fault_a, sizeof(fault_a)); - std::memcpy(memory + 0x8010, &fault_b, sizeof(fault_b)); - auto fault_a_desc = - MakeLinearDesc(base + 0x8000, sizeof(fault_a), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - auto fault_b_desc = fault_a_desc; - fault_b_desc.info.data.address = base + 0x8010; - const auto fault_a_image = texture_cache.FindImage(fault_a_desc); - const auto fault_b_image = texture_cache.FindImage(fault_b_desc); - (void)texture_cache.FindTexture(fault_a_image, fault_a_desc); - (void)texture_cache.FindTexture(fault_b_image, fault_b_desc); - texture_cache.MarkGpuWritten(fault_a_image); - texture_cache.MarkGpuWritten(fault_b_image); - Require(name, "per-image watcher installation", - texture_cache.GetImage(fault_a_image).IsTracked() && - texture_cache.GetImage(fault_b_image).IsTracked(), - "same-page images did not install independent write watchers"); - constexpr uint64_t padding_fault_offset = 0x8080; - uint32_t write_only_read_a = 0; - uint32_t write_only_read_b = 0; - std::memcpy(&write_only_read_a, memory + 0x8000, sizeof(write_only_read_a)); - std::memcpy(&write_only_read_b, memory + 0x8010, sizeof(write_only_read_b)); - Require(name, "write-only image read policy", - write_only_read_a == fault_a && write_only_read_b == fault_b && - texture_cache.GetImage(fault_a_image).IsGpuModified() && - texture_cache.GetImage(fault_b_image).IsGpuModified(), - "TextureCache claimed a CPU read fault instead of preserving " - "GPU image ownership"); - Require(name, "same-page image write invalidation", - resources.HandleFault(PageFaultAccess::Write, base + padding_fault_offset) && - texture_cache.GetImage(fault_a_image).IsGpuModified() && - texture_cache.GetImage(fault_b_image).IsGpuModified() && - !texture_cache.GetImage(fault_a_image).IsTracked() && - !texture_cache.GetImage(fault_b_image).IsTracked() && - texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() && - texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(), - "a byte-disjoint CPU write discarded authoritative images"); - const auto retracked_a = texture_cache.FindImage(fault_a_desc); - const auto retracked_b = texture_cache.FindImage(fault_b_desc); - (void)texture_cache.FindTexture(retracked_a, fault_a_desc); - (void)texture_cache.FindTexture(retracked_b, fault_b_desc); - auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer( - command, base + 0x8000, sizeof(fault_a), false, true, true); - if (fault_a_mirror.owner != nullptr) { - command.RetainResourceUntilFence(fault_a_mirror.owner); - } - Require(name, "same-page image re-track", - retracked_a == fault_a_image && retracked_b == fault_b_image && - texture_cache.GetImage(fault_a_image).IsTracked() && - texture_cache.GetImage(fault_b_image).IsTracked() && - !texture_cache.GetImage(fault_a_image).IsCpuDirty() && - !texture_cache.GetImage(fault_b_image).IsCpuDirty() && - fault_a_mirror.buffer != nullptr && - texture_cache.GetImage(fault_a_image).IsTracked() && - texture_cache.GetImage(fault_b_image).IsTracked() && - texture_cache.GetImage(fault_a_image).IsGpuModified() && - texture_cache.GetImage(fault_b_image).IsGpuModified(), - "copying one same-page image lost an authoritative owner"); - Require(name, "same-page survivor write fault", - resources.HandleFault(PageFaultAccess::Write, base + 0x8010) && - texture_cache.GetImage(fault_b_image).IsGpuModified() && - texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(), - "the surviving image was not protected after its alias retired"); - constexpr uint32_t fault_b_cpu = 0xa5a6a7a8u; - std::memcpy(memory + 0x8010, &fault_b_cpu, sizeof(fault_b_cpu)); - const auto refreshed_b = texture_cache.FindImage(fault_b_desc); - (void)texture_cache.FindTexture(refreshed_b, fault_b_desc); - Require(name, "same-page survivor refresh", - refreshed_b == fault_b_image && - !texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() && - texture_cache.GetImage(fault_b_image).IsGpuModified(), - "the surviving image could not reconcile its CPU write"); + const uint32_t fault_a = 0x01020304u; + const uint32_t fault_b = 0x11121314u; + std::memcpy(memory + 0x8000, &fault_a, sizeof(fault_a)); + std::memcpy(memory + 0x8010, &fault_b, sizeof(fault_b)); + auto fault_a_desc = + MakeLinearDesc(base + 0x8000, sizeof(fault_a), vk::Format::eR32Uint, + Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + auto fault_b_desc = fault_a_desc; + fault_b_desc.info.data.address = base + 0x8010; + const auto fault_a_image = texture_cache.FindImage(fault_a_desc); + const auto fault_b_image = texture_cache.FindImage(fault_b_desc); + (void)texture_cache.FindTexture(fault_a_image, fault_a_desc); + (void)texture_cache.FindTexture(fault_b_image, fault_b_desc); + texture_cache.MarkGpuWritten(fault_a_image); + texture_cache.MarkGpuWritten(fault_b_image); + Require(name, "per-image watcher installation", + texture_cache.GetImage(fault_a_image).IsTracked() && + texture_cache.GetImage(fault_b_image).IsTracked(), + "same-page images did not install independent write watchers"); + constexpr uint64_t padding_fault_offset = 0x8080; + uint32_t write_only_read_a = 0; + uint32_t write_only_read_b = 0; + std::memcpy(&write_only_read_a, memory + 0x8000, + sizeof(write_only_read_a)); + std::memcpy(&write_only_read_b, memory + 0x8010, + sizeof(write_only_read_b)); + Require(name, "write-only image read policy", + write_only_read_a == fault_a && write_only_read_b == fault_b && + texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "TextureCache claimed a CPU read fault instead of preserving " + "GPU image ownership"); + Require(name, "same-page image write invalidation", + resources.HandleFault(PageFaultAccess::Write, + base + padding_fault_offset) && + texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsGpuModified() && + !texture_cache.GetImage(fault_a_image).IsTracked() && + !texture_cache.GetImage(fault_b_image).IsTracked() && + texture_cache.GetImage(fault_a_image).IsMaybeCpuDirty() && + texture_cache.GetImage(fault_b_image).IsMaybeCpuDirty(), + "a byte-disjoint CPU write discarded authoritative images"); + const auto retracked_a = texture_cache.FindImage(fault_a_desc); + const auto retracked_b = texture_cache.FindImage(fault_b_desc); + (void)texture_cache.FindTexture(retracked_a, fault_a_desc); + (void)texture_cache.FindTexture(retracked_b, fault_b_desc); + auto fault_a_mirror = resources.GetBufferCache().ObtainBuffer( + command, base + 0x8000, sizeof(fault_a), false, true, true); + if (fault_a_mirror.owner != nullptr) { + command.RetainResourceUntilFence(fault_a_mirror.owner); + } + Require(name, "same-page image re-track", + retracked_a == fault_a_image && retracked_b == fault_b_image && + texture_cache.GetImage(fault_a_image).IsTracked() && + texture_cache.GetImage(fault_b_image).IsTracked() && + !texture_cache.GetImage(fault_a_image).IsCpuDirty() && + !texture_cache.GetImage(fault_b_image).IsCpuDirty() && + fault_a_mirror.buffer != nullptr && + texture_cache.GetImage(fault_a_image).IsTracked() && + texture_cache.GetImage(fault_b_image).IsTracked() && + texture_cache.GetImage(fault_a_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "copying one same-page image lost an authoritative owner"); + Require(name, "same-page survivor write fault", + resources.HandleFault(PageFaultAccess::Write, base + 0x8010) && + texture_cache.GetImage(fault_b_image).IsGpuModified() && + texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty(), + "the surviving image was not protected after its alias retired"); + constexpr uint32_t fault_b_cpu = 0xa5a6a7a8u; + std::memcpy(memory + 0x8010, &fault_b_cpu, sizeof(fault_b_cpu)); + const auto refreshed_b = texture_cache.FindImage(fault_b_desc); + (void)texture_cache.FindTexture(refreshed_b, fault_b_desc); + Require( + name, "same-page survivor refresh", + refreshed_b == fault_b_image && + !texture_cache.GetImage(fault_b_image).IsDefinitelyCpuDirty() && + texture_cache.GetImage(fault_b_image).IsGpuModified(), + "the surviving image could not reconcile its CPU write"); - constexpr uint64_t publish_image_offset = 0x26000; - constexpr uint64_t publish_buffer_offset = 0x26010; - constexpr uint32_t publish_image_value = 0x31415926u; - constexpr uint32_t publish_buffer_value = 0x27182818u; - std::memcpy(memory + publish_image_offset, &publish_image_value, - sizeof(publish_image_value)); - auto publish_image_desc = MakeLinearDesc( - base + publish_image_offset, sizeof(publish_image_value), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto publish_image = texture_cache.FindImage(publish_image_desc); - (void)texture_cache.FindTexture(publish_image, publish_image_desc); - texture_cache.MarkGpuWritten(publish_image); - resources.GetBufferCache().FillBuffer( - base + publish_buffer_offset, sizeof(publish_buffer_value), publish_buffer_value); - auto publish_replacement_desc = publish_image_desc; - publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat; - publish_replacement_desc.info.guest_format = - Prospero::BufferFormat::k32Float; - publish_replacement_desc.view_info.format = vk::Format::eR32Sfloat; - const auto publish_replacement = - texture_cache.FindImage(publish_replacement_desc, true); - Require(name, "exact-disjoint image replacement", - publish_replacement && publish_replacement != publish_image, - "exact-format lookup did not replace the image"); - Require(name, "exact-disjoint image publication", - !resources.GetBufferCache().HasGpuDirtyBytes(base + publish_buffer_offset, - sizeof(publish_buffer_value)), - "clean neighboring buffer unexpectedly became GPU-owned"); - uint32_t published_image_backing = 0; - uint32_t published_buffer_backing = 0; - std::memcpy(&published_image_backing, memory + publish_image_offset, - sizeof(published_image_backing)); - std::memcpy(&published_buffer_backing, memory + publish_buffer_offset, - sizeof(published_buffer_backing)); - Require(name, "exact-disjoint publication contents", - published_image_backing == publish_image_value && - published_buffer_backing == publish_buffer_value, - "image publication or neighboring buffer readback changed bytes"); + constexpr uint64_t publish_image_offset = 0x26000; + constexpr uint64_t publish_buffer_offset = 0x26010; + constexpr uint32_t publish_image_value = 0x31415926u; + constexpr uint32_t publish_buffer_value = 0x27182818u; + std::memcpy(memory + publish_image_offset, &publish_image_value, + sizeof(publish_image_value)); + auto publish_image_desc = MakeLinearDesc( + base + publish_image_offset, sizeof(publish_image_value), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto publish_image = texture_cache.FindImage(publish_image_desc); + (void)texture_cache.FindTexture(publish_image, publish_image_desc); + texture_cache.MarkGpuWritten(publish_image); + resources.GetBufferCache().FillBuffer(base + publish_buffer_offset, + sizeof(publish_buffer_value), + publish_buffer_value); + auto publish_replacement_desc = publish_image_desc; + publish_replacement_desc.info.pixel_format = vk::Format::eR32Sfloat; + publish_replacement_desc.info.guest_format = + Prospero::BufferFormat::k32Float; + publish_replacement_desc.view_info.format = vk::Format::eR32Sfloat; + const auto publish_replacement = + texture_cache.FindImage(publish_replacement_desc, true); + Require(name, "exact-disjoint image replacement", + publish_replacement && publish_replacement != publish_image, + "exact-format lookup did not replace the image"); + Require(name, "exact-disjoint image publication", + !resources.GetBufferCache().HasGpuDirtyBytes( + base + publish_buffer_offset, sizeof(publish_buffer_value)), + "clean neighboring buffer unexpectedly became GPU-owned"); + uint32_t published_image_backing = 0; + uint32_t published_buffer_backing = 0; + std::memcpy(&published_image_backing, memory + publish_image_offset, + sizeof(published_image_backing)); + std::memcpy(&published_buffer_backing, memory + publish_buffer_offset, + sizeof(published_buffer_backing)); + Require(name, "exact-disjoint publication contents", + published_image_backing == publish_image_value && + published_buffer_backing == publish_buffer_value, + "image publication or neighboring buffer readback changed bytes"); - for (const auto [offset, samples]: - std::array, 2> {{{0x10000, 2}, {0x11000, 4}}}) { - std::memset(memory + offset, 0, samples * sizeof(uint16_t)); - auto standalone_ms = - MakeLinearDesc(base + offset, samples * sizeof(uint16_t), vk::Format::eD16Unorm, - Prospero::BufferFormat::k16UNorm, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 2, samples); - standalone_ms.type = BindingType::DepthTarget; - standalone_ms.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - standalone_ms.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto image = texture_cache.FindImage(standalone_ms); - (void)texture_cache.FindDepthTarget(image, standalone_ms); - texture_cache.MarkGpuWritten(image); - uint16_t standalone_cpu_read = 1; - std::memcpy(&standalone_cpu_read, memory + offset, sizeof(standalone_cpu_read)); - Require(name, "standalone MSAA ownership", - texture_cache.GetImage(image).IsGpuModified() && standalone_cpu_read == 0 && - texture_cache.GetImage(image).IsGpuModified() && - resources.HandleFault(PageFaultAccess::Write, base + offset) && - texture_cache.GetImage(image).IsGpuModified() && - texture_cache.GetImage(image).IsDefinitelyCpuDirty(), - "fresh multisample target violated write-only ownership"); - } + for (const auto [offset, samples] : + std::array, 2>{ + {{0x10000, 2}, {0x11000, 4}}}) { + std::memset(memory + offset, 0, samples * sizeof(uint16_t)); + auto standalone_ms = MakeLinearDesc( + base + offset, samples * sizeof(uint16_t), vk::Format::eD16Unorm, + Prospero::BufferFormat::k16UNorm, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 2, samples); + standalone_ms.type = BindingType::DepthTarget; + standalone_ms.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + standalone_ms.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto image = texture_cache.FindImage(standalone_ms); + (void)texture_cache.FindDepthTarget(image, standalone_ms); + texture_cache.MarkGpuWritten(image); + uint16_t standalone_cpu_read = 1; + std::memcpy(&standalone_cpu_read, memory + offset, + sizeof(standalone_cpu_read)); + Require( + name, "standalone MSAA ownership", + texture_cache.GetImage(image).IsGpuModified() && + standalone_cpu_read == 0 && + texture_cache.GetImage(image).IsGpuModified() && + resources.HandleFault(PageFaultAccess::Write, base + offset) && + texture_cache.GetImage(image).IsGpuModified() && + texture_cache.GetImage(image).IsDefinitelyCpuDirty(), + "fresh multisample target violated write-only ownership"); + } - constexpr uint64_t metadata_data_a = 0x12000; - constexpr uint64_t metadata_data_b = 0x12100; - constexpr uint64_t metadata_a = 0x13000; - constexpr uint64_t metadata_b = 0x13100; - auto MakeMetadataDepth = [&](uint64_t data, uint64_t metadata) { - auto desc = MakeLinearDesc(base + data, sizeof(uint32_t), vk::Format::eD32Sfloat, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - desc.type = BindingType::DepthTarget; - desc.info.metadata.kind = ImageMetadataKind::Htile; - desc.info.metadata.range = {base + metadata, 0x80}; - desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - return desc; - }; - auto metadata_depth_a = MakeMetadataDepth(metadata_data_a, metadata_a); - auto metadata_depth_b = MakeMetadataDepth(metadata_data_b, metadata_b); - const auto metadata_depth_a_id = texture_cache.FindImage(metadata_depth_a); - const auto metadata_depth_b_id = texture_cache.FindImage(metadata_depth_b); - const auto metadata_depth_a_view = - texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a); - const auto metadata_depth_b_view = - texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b); - Require(name, "shared-page metadata state", - metadata_depth_a_view != nullptr && metadata_depth_b_view != nullptr && - texture_cache.ClearMeta(base + metadata_a) && - texture_cache.ClearMeta(base + metadata_b), - "shared-page metadata did not retain logical clear state"); - auto metadata_alias = - MakeLinearDesc(base + metadata_a, sizeof(uint32_t), vk::Format::eR32Sfloat, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - metadata_alias.type = BindingType::RenderTarget; - metadata_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - const auto metadata_alias_id = texture_cache.FindImage(metadata_alias, true); - const auto metadata_alias_view = - texture_cache.FindRenderTarget(metadata_alias_id, metadata_alias); - Require(name, "exact alias metadata coexistence", - metadata_alias_view != nullptr && - texture_cache.GetImage(metadata_depth_a_id).IsGpuModified() && - texture_cache.GetImage(metadata_depth_b_id).IsGpuModified() && - texture_cache.IsMeta(base + metadata_a) && - texture_cache.IsMetaCleared(base + metadata_a, 0) && - texture_cache.IsMeta(base + metadata_b) && - texture_cache.IsMetaCleared(base + metadata_b, 0), - "image discovery retired aliased metadata or its live depth image"); - Require(name, "metadata first-touch state", - texture_cache.TouchMeta(base + metadata_b, 0, true) && - texture_cache.IsMetaCleared(base + metadata_b, 0) && - texture_cache.TouchMeta(base + metadata_b, 0, false) && - !texture_cache.IsMetaCleared(base + metadata_b, 0), - "per-slice metadata update incorrectly required prior GPU ownership"); - resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), 0); - Require(name, "metadata buffer fill state", - texture_cache.IsMetaCleared(base + metadata_b, 0), - "BufferCache fill did not publish an exact-address metadata clear"); + constexpr uint64_t metadata_data_a = 0x12000; + constexpr uint64_t metadata_data_b = 0x12100; + constexpr uint64_t metadata_a = 0x13000; + constexpr uint64_t metadata_b = 0x13100; + auto MakeMetadataDepth = [&](uint64_t data, uint64_t metadata) { + auto desc = MakeLinearDesc( + base + data, sizeof(uint32_t), vk::Format::eD32Sfloat, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + desc.type = BindingType::DepthTarget; + desc.info.metadata.kind = ImageMetadataKind::Htile; + desc.info.metadata.range = {base + metadata, 0x80}; + desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + return desc; + }; + auto metadata_depth_a = MakeMetadataDepth(metadata_data_a, metadata_a); + auto metadata_depth_b = MakeMetadataDepth(metadata_data_b, metadata_b); + const auto metadata_depth_a_id = + texture_cache.FindImage(metadata_depth_a); + const auto metadata_depth_b_id = + texture_cache.FindImage(metadata_depth_b); + const auto metadata_depth_a_view = + texture_cache.FindDepthTarget(metadata_depth_a_id, metadata_depth_a); + const auto metadata_depth_b_view = + texture_cache.FindDepthTarget(metadata_depth_b_id, metadata_depth_b); + Require(name, "shared-page metadata state", + metadata_depth_a_view != nullptr && + metadata_depth_b_view != nullptr && + texture_cache.ClearMeta(base + metadata_a) && + texture_cache.ClearMeta(base + metadata_b), + "shared-page metadata did not retain logical clear state"); + auto metadata_alias = MakeLinearDesc( + base + metadata_a, sizeof(uint32_t), vk::Format::eR32Sfloat, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + metadata_alias.type = BindingType::RenderTarget; + metadata_alias.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + const auto metadata_alias_id = + texture_cache.FindImage(metadata_alias, true); + const auto metadata_alias_view = + texture_cache.FindRenderTarget(metadata_alias_id, metadata_alias); + Require( + name, "exact alias metadata coexistence", + metadata_alias_view != nullptr && + texture_cache.GetImage(metadata_depth_a_id).IsGpuModified() && + texture_cache.GetImage(metadata_depth_b_id).IsGpuModified() && + texture_cache.IsMeta(base + metadata_a) && + texture_cache.IsMetaCleared(base + metadata_a, 0) && + texture_cache.IsMeta(base + metadata_b) && + texture_cache.IsMetaCleared(base + metadata_b, 0), + "image discovery retired aliased metadata or its live depth image"); + Require( + name, "metadata first-touch state", + texture_cache.TouchMeta(base + metadata_b, 0, true) && + texture_cache.IsMetaCleared(base + metadata_b, 0) && + texture_cache.TouchMeta(base + metadata_b, 0, false) && + !texture_cache.IsMetaCleared(base + metadata_b, 0), + "per-slice metadata update incorrectly required prior GPU ownership"); + resources.GetBufferCache().FillBuffer(base + metadata_b, sizeof(uint32_t), + 0); + Require( + name, "metadata buffer fill state", + texture_cache.IsMetaCleared(base + metadata_b, 0), + "BufferCache fill did not publish an exact-address metadata clear"); - constexpr uint64_t partial_unmap_image_offset = 0x2700000; - auto partial_unmap_image = - MakeLinearDesc(base + partial_unmap_image_offset, 0x2000, vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {2048, 1, 1}, 1, 4, 1); - const auto partial_unmap_image_id = texture_cache.FindImage(partial_unmap_image); - texture_cache.UnmapMemory(partial_unmap_image.info.data.address, 0x1000); - Require(name, "partial image unmap tracking", - partial_unmap_image_id && - !texture_cache.FindImageFromRange(partial_unmap_image.info.data.address, - 0x2000, false), - "partial unmap left the deleted image's mapped tail tracked"); + constexpr uint64_t partial_unmap_image_offset = 0x2700000; + auto partial_unmap_image = + MakeLinearDesc(base + partial_unmap_image_offset, 0x2000, + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {2048, 1, 1}, 1, 4, 1); + const auto partial_unmap_image_id = + texture_cache.FindImage(partial_unmap_image); + texture_cache.UnmapMemory(partial_unmap_image.info.data.address, 0x1000); + Require(name, "partial image unmap tracking", + partial_unmap_image_id && + !texture_cache.FindImageFromRange( + partial_unmap_image.info.data.address, 0x2000, false), + "partial unmap left the deleted image's mapped tail tracked"); - constexpr uint64_t unformatted_alias_offset = 0x2500000; - auto unformatted_alias = MakeLinearDesc( - base + unformatted_alias_offset, sizeof(uint32_t), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto unformatted_alias_image = texture_cache.FindImage(unformatted_alias); - (void)texture_cache.FindTexture(unformatted_alias_image, unformatted_alias); - texture_cache.MarkGpuWritten(unformatted_alias_image); - auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer( - command, unformatted_alias.info.data.address, unformatted_alias.info.data.size, - false, true, false); - Require(name, "unformatted buffer/image alias", - unformatted_alias_buffer.buffer != nullptr && - texture_cache.GetImage(unformatted_alias_image).IsGpuModified() && - !texture_cache.GetImage(unformatted_alias_image).IsBufferModified(), - "read-only unformatted buffer alias did not follow ordinary " - "BufferCache acquisition"); - if (unformatted_alias_buffer.owner != nullptr) { - command.RetainResourceUntilFence(unformatted_alias_buffer.owner); - } + constexpr uint64_t unformatted_alias_offset = 0x2500000; + auto unformatted_alias = + MakeLinearDesc(base + unformatted_alias_offset, sizeof(uint32_t), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto unformatted_alias_image = + texture_cache.FindImage(unformatted_alias); + (void)texture_cache.FindTexture(unformatted_alias_image, + unformatted_alias); + texture_cache.MarkGpuWritten(unformatted_alias_image); + auto unformatted_alias_buffer = resources.GetBufferCache().ObtainBuffer( + command, unformatted_alias.info.data.address, + unformatted_alias.info.data.size, false, true, false); + Require( + name, "unformatted buffer/image alias", + unformatted_alias_buffer.buffer != nullptr && + texture_cache.GetImage(unformatted_alias_image).IsGpuModified() && + !texture_cache.GetImage(unformatted_alias_image) + .IsBufferModified(), + "read-only unformatted buffer alias did not follow ordinary " + "BufferCache acquisition"); + if (unformatted_alias_buffer.owner != nullptr) { + command.RetainResourceUntilFence(unformatted_alias_buffer.owner); + } - constexpr uint64_t layered_offset = 0x14000; - constexpr uint64_t layered_guest_size = 0x800; - const std::array layered_values {0.0f, 0.125f, 0.25f, 0.375f, 0.5f, - 0.625f, 0.75f, 0.875f, 1.0f, 0.0625f}; - std::memset(memory + layered_offset, 0, layered_guest_size); - const auto layered_layout = TextureCalcUploadLayout( - Prospero::BufferFormat::k32Float, 2, 2, 2, 2, - Prospero::TileMode::kLinear, layered_guest_size, true, - false, "UnifiedTextureCacheFlow"); - const auto layered_upload_regions = TextureBuildImageCopies(layered_layout); - Require(name, "layered guest layout", layered_upload_regions.size() == 4, - "unexpected layered/mipped upload-region count"); - for (const auto& region: layered_upload_regions) { - for (uint32_t y = 0; y < region.imageExtent.height; y++) { - for (uint32_t x = 0; x < region.imageExtent.width; x++) { - const uint32_t logical = - region.imageSubresource.mipLevel == 0 - ? region.imageSubresource.baseArrayLayer * 4 + y * 2 + x - : 8 + region.imageSubresource.baseArrayLayer; - const uint64_t byte_offset = - region.bufferOffset + - (static_cast(y) * region.bufferRowLength + x) * sizeof(float); - Require(name, "layered guest layout bounds", - byte_offset + sizeof(float) <= layered_guest_size, - "layered guest texel lies outside its backing range"); - std::memcpy(memory + layered_offset + byte_offset, &layered_values[logical], - sizeof(float)); - } - } - } - auto layered_color = - MakeLinearDesc(base + layered_offset, layered_guest_size, vk::Format::eR32Sfloat, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {2, 2, 1}, 2, 4, 1); - layered_color.info.resources.levels = 2; - layered_color.info.mip_layout[0] = {0, 32, 2, 2}; - layered_color.info.mip_layout[1] = {32, 8, 1, 1}; - layered_color.view_info.level_count = 2; - const auto layered_color_image = texture_cache.FindImage(layered_color); - (void)texture_cache.FindTexture(layered_color_image, layered_color); - texture_cache.MarkGpuWritten(layered_color_image); - auto layered_depth = layered_color; - layered_depth.type = BindingType::DepthTarget; - layered_depth.info.resources = {1, 4}; - layered_depth.view_info.level_count = 1; - layered_depth.info.pixel_format = vk::Format::eD32Sfloat; - layered_depth.view_info.format = vk::Format::eD32Sfloat; - layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - layered_depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto layered_depth_image = texture_cache.FindImage(layered_depth); - const auto& layered_native = texture_cache.GetImage(layered_depth_image); - Require(name, "layered mipped depth alias", - layered_depth_image != layered_color_image && - layered_native.backing.layers == 2 && - layered_native.backing.mip_levels == 2 && - layered_native.info.resources == layered_color.info.resources && - layered_native.IsGpuModified(), - "depth/color conversion did not use the lexicographic " - "resource maximum"); - const volatile auto layered_guest_byte = - *reinterpret_cast(memory + layered_offset); - (void)layered_guest_byte; - Require(name, "layered depth write-only read policy", - texture_cache.GetImage(layered_depth_image).IsGpuModified(), - "a CPU read invalidated layered depth GPU ownership"); + constexpr uint64_t layered_offset = 0x14000; + constexpr uint64_t layered_guest_size = 0x800; + const std::array layered_values{0.0f, 0.125f, 0.25f, 0.375f, + 0.5f, 0.625f, 0.75f, 0.875f, + 1.0f, 0.0625f}; + std::memset(memory + layered_offset, 0, layered_guest_size); + const auto layered_layout = TextureCalcUploadLayout( + Prospero::BufferFormat::k32Float, 2, 2, 2, 2, + Prospero::TileMode::kLinear, layered_guest_size, true, false, + "UnifiedTextureCacheFlow"); + const auto layered_upload_regions = + TextureBuildImageCopies(layered_layout); + Require(name, "layered guest layout", layered_upload_regions.size() == 4, + "unexpected layered/mipped upload-region count"); + for (const auto ®ion : layered_upload_regions) { + for (uint32_t y = 0; y < region.imageExtent.height; y++) { + for (uint32_t x = 0; x < region.imageExtent.width; x++) { + const uint32_t logical = + region.imageSubresource.mipLevel == 0 + ? region.imageSubresource.baseArrayLayer * 4 + y * 2 + x + : 8 + region.imageSubresource.baseArrayLayer; + const uint64_t byte_offset = + region.bufferOffset + + (static_cast(y) * region.bufferRowLength + x) * + sizeof(float); + Require(name, "layered guest layout bounds", + byte_offset + sizeof(float) <= layered_guest_size, + "layered guest texel lies outside its backing range"); + std::memcpy(memory + layered_offset + byte_offset, + &layered_values[logical], sizeof(float)); + } + } + } + auto layered_color = MakeLinearDesc( + base + layered_offset, layered_guest_size, vk::Format::eR32Sfloat, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {2, 2, 1}, 2, 4, 1); + layered_color.info.resources.levels = 2; + layered_color.info.mip_layout[0] = {0, 32, 2, 2}; + layered_color.info.mip_layout[1] = {32, 8, 1, 1}; + layered_color.view_info.level_count = 2; + const auto layered_color_image = texture_cache.FindImage(layered_color); + (void)texture_cache.FindTexture(layered_color_image, layered_color); + texture_cache.MarkGpuWritten(layered_color_image); + auto layered_depth = layered_color; + layered_depth.type = BindingType::DepthTarget; + layered_depth.info.resources = {1, 4}; + layered_depth.view_info.level_count = 1; + layered_depth.info.pixel_format = vk::Format::eD32Sfloat; + layered_depth.view_info.format = vk::Format::eD32Sfloat; + layered_depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + layered_depth.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto layered_depth_image = texture_cache.FindImage(layered_depth); + const auto &layered_native = texture_cache.GetImage(layered_depth_image); + Require(name, "layered mipped depth alias", + layered_depth_image != layered_color_image && + layered_native.backing.layers == 2 && + layered_native.backing.mip_levels == 2 && + layered_native.info.resources == + layered_color.info.resources && + layered_native.IsGpuModified(), + "depth/color conversion did not use the lexicographic " + "resource maximum"); + const volatile auto layered_guest_byte = + *reinterpret_cast(memory + layered_offset); + (void)layered_guest_byte; + Require(name, "layered depth write-only read policy", + texture_cache.GetImage(layered_depth_image).IsGpuModified(), + "a CPU read invalidated layered depth GPU ownership"); - constexpr uint64_t replacement_offset = 0x18000; - constexpr uint64_t replacement_size = 0x2000; - constexpr uint32_t replacement_pixel = 0xcafebabeu; - constexpr uint32_t replacement_tail = 0x0badf00du; - std::memset(memory + replacement_offset, 0, replacement_size); - std::memcpy(memory + replacement_offset, &replacement_pixel, sizeof(replacement_pixel)); - std::memcpy(memory + replacement_offset + replacement_size - sizeof(replacement_tail), - &replacement_tail, sizeof(replacement_tail)); - auto replacement_source = - MakeLinearDesc(base + replacement_offset, sizeof(uint32_t), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto replacement_source_image = texture_cache.FindImage(replacement_source); - (void)texture_cache.FindTexture(replacement_source_image, replacement_source); - texture_cache.MarkGpuWritten(replacement_source_image); - auto replacement_desc = - MakeLinearDesc(base + replacement_offset, replacement_size, vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); - const auto replacement = texture_cache.FindImage(replacement_desc); - const bool replacement_changed = replacement != replacement_source_image; - const bool replacement_gpu_before = texture_cache.GetImage(replacement).IsGpuModified(); - const bool replacement_fault = - resources.HandleFault(PageFaultAccess::Write, base + replacement_offset + 0x1000); - const bool replacement_gpu_after = texture_cache.GetImage(replacement).IsGpuModified(); - uint32_t replacement_pixel_after = 0; - uint32_t replacement_tail_after = 0; - std::memcpy(&replacement_pixel_after, memory + replacement_offset, - sizeof(replacement_pixel_after)); - std::memcpy(&replacement_tail_after, - memory + replacement_offset + replacement_size - - sizeof(replacement_tail_after), - sizeof(replacement_tail_after)); - Require(name, "replacement tracking", - replacement_changed && replacement_gpu_before && replacement_fault && - replacement_gpu_after && - texture_cache.GetImage(replacement).IsDefinitelyCpuDirty() && - replacement_pixel_after == replacement_pixel && - replacement_tail_after == replacement_tail, - "replacement write invalidation lost native authority or guest " - "backing"); + constexpr uint64_t replacement_offset = 0x18000; + constexpr uint64_t replacement_size = 0x2000; + constexpr uint32_t replacement_pixel = 0xcafebabeu; + constexpr uint32_t replacement_tail = 0x0badf00du; + std::memset(memory + replacement_offset, 0, replacement_size); + std::memcpy(memory + replacement_offset, &replacement_pixel, + sizeof(replacement_pixel)); + std::memcpy(memory + replacement_offset + replacement_size - + sizeof(replacement_tail), + &replacement_tail, sizeof(replacement_tail)); + auto replacement_source = + MakeLinearDesc(base + replacement_offset, sizeof(uint32_t), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto replacement_source_image = + texture_cache.FindImage(replacement_source); + (void)texture_cache.FindTexture(replacement_source_image, + replacement_source); + texture_cache.MarkGpuWritten(replacement_source_image); + auto replacement_desc = + MakeLinearDesc(base + replacement_offset, replacement_size, + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 2, 4, 1); + const auto replacement = texture_cache.FindImage(replacement_desc); + const bool replacement_changed = replacement != replacement_source_image; + const bool replacement_gpu_before = + texture_cache.GetImage(replacement).IsGpuModified(); + const bool replacement_fault = resources.HandleFault( + PageFaultAccess::Write, base + replacement_offset + 0x1000); + const bool replacement_gpu_after = + texture_cache.GetImage(replacement).IsGpuModified(); + uint32_t replacement_pixel_after = 0; + uint32_t replacement_tail_after = 0; + std::memcpy(&replacement_pixel_after, memory + replacement_offset, + sizeof(replacement_pixel_after)); + std::memcpy(&replacement_tail_after, + memory + replacement_offset + replacement_size - + sizeof(replacement_tail_after), + sizeof(replacement_tail_after)); + Require(name, "replacement tracking", + replacement_changed && replacement_gpu_before && + replacement_fault && replacement_gpu_after && + texture_cache.GetImage(replacement).IsDefinitelyCpuDirty() && + replacement_pixel_after == replacement_pixel && + replacement_tail_after == replacement_tail, + "replacement write invalidation lost native authority or guest " + "backing"); - const uint32_t compressed_value = 0x55667788u; - std::memcpy(memory + 0xc000, &compressed_value, sizeof(compressed_value)); - auto compressed_desc = - MakeLinearDesc(base + 0xc000, sizeof(compressed_value), vk::Format::eR8G8B8A8Srgb, - Prospero::BufferFormat::k8_8_8_8Srgb, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - compressed_desc.type = BindingType::RenderTarget; - compressed_desc.info.metadata.kind = ImageMetadataKind::Dcc; - compressed_desc.info.metadata.range = {base + 0xd000, 0}; - compressed_desc.info.metadata.compression = VideoOutCompression::Dcc256_256_0; - compressed_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - const auto compressed_image = texture_cache.FindImage(compressed_desc); - (void)texture_cache.FindRenderTarget(compressed_image, compressed_desc); - uint32_t compressed_cpu_read = 0; - std::memcpy(&compressed_cpu_read, memory + 0xc000, sizeof(compressed_cpu_read)); - Require(name, "compressed write-only read policy", - compressed_cpu_read == compressed_value && - texture_cache.GetImage(compressed_image).IsGpuModified(), - "compressed image incorrectly claimed a CPU read fault"); - Require(name, "compressed download rejection", - !TextureCacheTestAccess::TryDownload(texture_cache, compressed_image) && - texture_cache.GetImage(compressed_image).IsGpuModified() && - !texture_cache.GetImage(compressed_image).IsBufferModified(), - "a compressed image escaped the unified download guard"); - auto compressed_video_desc = compressed_desc; - compressed_video_desc.type = BindingType::VideoOut; - compressed_video_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto compressed_video = texture_cache.FindImage(compressed_video_desc); - const auto& compressed_video_image = texture_cache.GetImage(compressed_video); - Require(name, "cache-only video-out discovery", - compressed_video == compressed_image && - !compressed_video_image.binding.is_bound && - !compressed_video_image.binding.is_target && - !compressed_video_image.binding.needs_rebind && - !compressed_video_image.usage.video_out, - "FindImage claimed caller-owned video-out usage or binding state"); + const uint32_t compressed_value = 0x55667788u; + std::memcpy(memory + 0xc000, &compressed_value, sizeof(compressed_value)); + auto compressed_desc = MakeLinearDesc( + base + 0xc000, sizeof(compressed_value), vk::Format::eR8G8B8A8Srgb, + Prospero::BufferFormat::k8_8_8_8Srgb, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + compressed_desc.type = BindingType::RenderTarget; + compressed_desc.info.metadata.kind = ImageMetadataKind::Dcc; + compressed_desc.info.metadata.range = {base + 0xd000, 0}; + compressed_desc.info.metadata.compression = + VideoOutCompression::Dcc256_256_0; + compressed_desc.view_info.usage = + vk::ImageUsageFlagBits::eColorAttachment; + const auto compressed_image = texture_cache.FindImage(compressed_desc); + (void)texture_cache.FindRenderTarget(compressed_image, compressed_desc); + uint32_t compressed_cpu_read = 0; + std::memcpy(&compressed_cpu_read, memory + 0xc000, + sizeof(compressed_cpu_read)); + Require(name, "compressed write-only read policy", + compressed_cpu_read == compressed_value && + texture_cache.GetImage(compressed_image).IsGpuModified(), + "compressed image incorrectly claimed a CPU read fault"); + Require(name, "compressed download rejection", + !TextureCacheTestAccess::TryDownload(texture_cache, + compressed_image) && + texture_cache.GetImage(compressed_image).IsGpuModified() && + !texture_cache.GetImage(compressed_image).IsBufferModified(), + "a compressed image escaped the unified download guard"); + auto compressed_video_desc = compressed_desc; + compressed_video_desc.type = BindingType::VideoOut; + compressed_video_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto compressed_video = + texture_cache.FindImage(compressed_video_desc); + const auto &compressed_video_image = + texture_cache.GetImage(compressed_video); + Require( + name, "cache-only video-out discovery", + compressed_video == compressed_image && + !compressed_video_image.binding.is_bound && + !compressed_video_image.binding.is_target && + !compressed_video_image.binding.needs_rebind && + !compressed_video_image.usage.video_out, + "FindImage claimed caller-owned video-out usage or binding state"); - constexpr uint64_t mixed_source_offset = 0x20000; - constexpr uint64_t mixed_source_size = 0x1004; - constexpr uint32_t mixed_source_width = 1025; - constexpr uint32_t mixed_cpu_value = 0x1234abcdu; - constexpr uint32_t mixed_gpu_value = 0x9876fedcu; - auto mixed_owner = resources.GetBufferCache().ObtainBuffer( - command, base + mixed_source_offset, mixed_source_size, true, true); - Require(name, "mixed-page source allocation", mixed_owner.owner != nullptr, - "mixed CPU/GPU image source did not create a containing buffer"); - command.RetainResourceUntilFence(mixed_owner.owner); - Require(name, "mixed-page CPU write fault", - resources.HandleFault(PageFaultAccess::Write, base + mixed_source_offset), - "mixed image source could not dirty its first page"); - std::memcpy(memory + mixed_source_offset, &mixed_cpu_value, sizeof(mixed_cpu_value)); - resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, - sizeof(mixed_gpu_value), mixed_gpu_value); - auto mixed_source_desc = - MakeLinearDesc(base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {mixed_source_width, 1, 1}, 1, 4, 1); - const auto mixed_source_image = texture_cache.FindImage(mixed_source_desc); - (void)texture_cache.FindTexture(mixed_source_image, mixed_source_desc); - Require(name, "mixed-page image upload", - !texture_cache.GetImage(mixed_source_image).IsGpuModified(), - "clean Buffer source manufactured GPU image ownership"); - auto mixed_source_readback = - CreateHostBuffer(name, mixed_source_size, vk::BufferUsageFlagBits::eTransferDst, - std::vector(mixed_source_size / sizeof(u32), 0)); - auto& mixed_source_native = texture_cache.GetImage(mixed_source_image); - mixed_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command.Handle()); - vk::BufferImageCopy mixed_source_copy {}; - mixed_source_copy.bufferRowLength = mixed_source_width; - mixed_source_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - mixed_source_copy.imageSubresource.layerCount = 1; - mixed_source_copy.imageExtent = {mixed_source_width, 1, 1}; - command.Handle().copyImageToBuffer(mixed_source_native.backing.image, - vk::ImageLayout::eTransferSrcOptimal, - mixed_source_readback.buffer, 1, &mixed_source_copy); - HostReadBarrier(mixed_source_readback.buffer, mixed_source_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); + constexpr uint64_t mixed_source_offset = 0x20000; + constexpr uint64_t mixed_source_size = 0x1004; + constexpr uint32_t mixed_source_width = 1025; + constexpr uint32_t mixed_cpu_value = 0x1234abcdu; + constexpr uint32_t mixed_gpu_value = 0x9876fedcu; + auto mixed_owner = resources.GetBufferCache().ObtainBuffer( + command, base + mixed_source_offset, mixed_source_size, true, true); + Require(name, "mixed-page source allocation", + mixed_owner.owner != nullptr, + "mixed CPU/GPU image source did not create a containing buffer"); + command.RetainResourceUntilFence(mixed_owner.owner); + Require(name, "mixed-page CPU write fault", + resources.HandleFault(PageFaultAccess::Write, + base + mixed_source_offset), + "mixed image source could not dirty its first page"); + std::memcpy(memory + mixed_source_offset, &mixed_cpu_value, + sizeof(mixed_cpu_value)); + resources.GetBufferCache().FillBuffer(base + mixed_source_offset + 0x1000, + sizeof(mixed_gpu_value), + mixed_gpu_value); + auto mixed_source_desc = MakeLinearDesc( + base + mixed_source_offset, mixed_source_size, vk::Format::eR32Uint, + Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D, + {mixed_source_width, 1, 1}, 1, 4, 1); + const auto mixed_source_image = + texture_cache.FindImage(mixed_source_desc); + (void)texture_cache.FindTexture(mixed_source_image, mixed_source_desc); + Require(name, "mixed-page image upload", + !texture_cache.GetImage(mixed_source_image).IsGpuModified(), + "clean Buffer source manufactured GPU image ownership"); + auto mixed_source_readback = CreateHostBuffer( + name, mixed_source_size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(mixed_source_size / sizeof(u32), 0)); + auto &mixed_source_native = texture_cache.GetImage(mixed_source_image); + mixed_source_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + vk::BufferImageCopy mixed_source_copy{}; + mixed_source_copy.bufferRowLength = mixed_source_width; + mixed_source_copy.imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eColor; + mixed_source_copy.imageSubresource.layerCount = 1; + mixed_source_copy.imageExtent = {mixed_source_width, 1, 1}; + command.Handle().copyImageToBuffer(mixed_source_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, + mixed_source_readback.buffer, 1, + &mixed_source_copy); + HostReadBarrier(mixed_source_readback.buffer, mixed_source_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); - constexpr uint64_t byte_mirror_page_offset = 0x24000; - constexpr uint64_t byte_mirror_offset = byte_mirror_page_offset + 1; - const std::array byte_mirror_guest {0xacu, 0x5au, 0xbdu, 0xceu}; - std::memcpy(memory + byte_mirror_page_offset, byte_mirror_guest.data(), - byte_mirror_guest.size()); - auto byte_mirror_desc = - MakeLinearDesc(base + byte_mirror_offset, 1, vk::Format::eR8Unorm, - Prospero::BufferFormat::k8UNorm, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 1, 1); - const auto byte_mirror_image = texture_cache.FindImage(byte_mirror_desc); - (void)texture_cache.FindTexture(byte_mirror_image, byte_mirror_desc); - texture_cache.MarkGpuWritten(byte_mirror_image); - auto byte_mirror = resources.GetBufferCache().ObtainBuffer( - command, base + byte_mirror_offset, 1, false, true, true); - Require(name, "byte image mirror", - byte_mirror.buffer != nullptr && - !texture_cache.GetImage(byte_mirror_image).IsBufferModified() && - texture_cache.GetImage(byte_mirror_image).IsGpuModified(), - "one-byte image copy transferred cache ownership"); - if (byte_mirror.owner != nullptr) { - command.RetainResourceUntilFence(byte_mirror.owner); - } + constexpr uint64_t byte_mirror_page_offset = 0x24000; + constexpr uint64_t byte_mirror_offset = byte_mirror_page_offset + 1; + const std::array byte_mirror_guest{0xacu, 0x5au, 0xbdu, + 0xceu}; + std::memcpy(memory + byte_mirror_page_offset, byte_mirror_guest.data(), + byte_mirror_guest.size()); + auto byte_mirror_desc = + MakeLinearDesc(base + byte_mirror_offset, 1, vk::Format::eR8Unorm, + Prospero::BufferFormat::k8UNorm, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 1, 1); + const auto byte_mirror_image = texture_cache.FindImage(byte_mirror_desc); + (void)texture_cache.FindTexture(byte_mirror_image, byte_mirror_desc); + texture_cache.MarkGpuWritten(byte_mirror_image); + auto byte_mirror = resources.GetBufferCache().ObtainBuffer( + command, base + byte_mirror_offset, 1, false, true, true); + Require( + name, "byte image mirror", + byte_mirror.buffer != nullptr && + !texture_cache.GetImage(byte_mirror_image).IsBufferModified() && + texture_cache.GetImage(byte_mirror_image).IsGpuModified(), + "one-byte image copy transferred cache ownership"); + if (byte_mirror.owner != nullptr) { + command.RetainResourceUntilFence(byte_mirror.owner); + } - const std::array bgra16_guest {0x3c00u, 0x4000u, 0x4200u, 0x4400u}; - std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest)); - auto bgra16_desc = - MakeLinearDesc(base + 0xb000, sizeof(bgra16_guest), vk::Format::eR16G16B16A16Sfloat, - Prospero::BufferFormat::k16_16_16_16Float, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1); - bgra16_desc.type = BindingType::RenderTarget; - bgra16_desc.info.bgra16 = true; - bgra16_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - const auto bgra16_image = texture_cache.FindImage(bgra16_desc); - (void)texture_cache.FindRenderTarget(bgra16_image, bgra16_desc); - auto bgra16_readback = CreateHostBuffer(name, sizeof(bgra16_guest), - vk::BufferUsageFlagBits::eTransferDst, {0, 0}); - auto& bgra16_native = texture_cache.GetImage(bgra16_image); - bgra16_native.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command.Handle()); - vk::BufferImageCopy bgra16_copy {}; - bgra16_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - bgra16_copy.imageSubresource.layerCount = 1; - bgra16_copy.imageExtent = {1, 1, 1}; - command.Handle().copyImageToBuffer(bgra16_native.backing.image, - vk::ImageLayout::eTransferSrcOptimal, - bgra16_readback.buffer, 1, &bgra16_copy); - HostReadBarrier(bgra16_readback.buffer, bgra16_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); + const std::array bgra16_guest{0x3c00u, 0x4000u, 0x4200u, + 0x4400u}; + std::memcpy(memory + 0xb000, bgra16_guest.data(), sizeof(bgra16_guest)); + auto bgra16_desc = MakeLinearDesc( + base + 0xb000, sizeof(bgra16_guest), vk::Format::eR16G16B16A16Sfloat, + Prospero::BufferFormat::k16_16_16_16Float, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 8, 1); + bgra16_desc.type = BindingType::RenderTarget; + bgra16_desc.info.bgra16 = true; + bgra16_desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + const auto bgra16_image = texture_cache.FindImage(bgra16_desc); + (void)texture_cache.FindRenderTarget(bgra16_image, bgra16_desc); + auto bgra16_readback = + CreateHostBuffer(name, sizeof(bgra16_guest), + vk::BufferUsageFlagBits::eTransferDst, {0, 0}); + auto &bgra16_native = texture_cache.GetImage(bgra16_image); + bgra16_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + vk::BufferImageCopy bgra16_copy{}; + bgra16_copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + bgra16_copy.imageSubresource.layerCount = 1; + bgra16_copy.imageExtent = {1, 1, 1}; + command.Handle().copyImageToBuffer( + bgra16_native.backing.image, vk::ImageLayout::eTransferSrcOptimal, + bgra16_readback.buffer, 1, &bgra16_copy); + HostReadBarrier(bgra16_readback.buffer, bgra16_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); - auto layered_readback = CreateHostBuffer( - name, sizeof(layered_values), vk::BufferUsageFlagBits::eTransferDst, - std::vector(sizeof(layered_values) / sizeof(u32), 0)); - auto& layered_depth_native = texture_cache.GetImage(layered_depth_image); - layered_depth_native.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, command.Handle()); - std::array layered_copies {}; - layered_copies[0].bufferOffset = 0; - layered_copies[0].imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth; - layered_copies[0].imageSubresource.mipLevel = 0; - layered_copies[0].imageSubresource.layerCount = 2; - layered_copies[0].imageExtent = {2, 2, 1}; - layered_copies[1].bufferOffset = 32; - layered_copies[1].imageSubresource.aspectMask = vk::ImageAspectFlagBits::eDepth; - layered_copies[1].imageSubresource.mipLevel = 1; - layered_copies[1].imageSubresource.layerCount = 2; - layered_copies[1].imageExtent = {1, 1, 1}; - command.Handle().copyImageToBuffer( - layered_depth_native.backing.image, vk::ImageLayout::eTransferSrcOptimal, - layered_readback.buffer, static_cast(layered_copies.size()), - layered_copies.data()); - HostReadBarrier(layered_readback.buffer, layered_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); + auto layered_readback = CreateHostBuffer( + name, sizeof(layered_values), vk::BufferUsageFlagBits::eTransferDst, + std::vector(sizeof(layered_values) / sizeof(u32), 0)); + auto &layered_depth_native = texture_cache.GetImage(layered_depth_image); + layered_depth_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + command.Handle()); + std::array layered_copies{}; + layered_copies[0].bufferOffset = 0; + layered_copies[0].imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eDepth; + layered_copies[0].imageSubresource.mipLevel = 0; + layered_copies[0].imageSubresource.layerCount = 2; + layered_copies[0].imageExtent = {2, 2, 1}; + layered_copies[1].bufferOffset = 32; + layered_copies[1].imageSubresource.aspectMask = + vk::ImageAspectFlagBits::eDepth; + layered_copies[1].imageSubresource.mipLevel = 1; + layered_copies[1].imageSubresource.layerCount = 2; + layered_copies[1].imageExtent = {1, 1, 1}; + command.Handle().copyImageToBuffer( + layered_depth_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, layered_readback.buffer, + static_cast(layered_copies.size()), layered_copies.data()); + HostReadBarrier(layered_readback.buffer, layered_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); - vk::ShaderModuleCreateInfo module_info {}; - module_info.sType = vk::StructureType::eShaderModuleCreateInfo; - module_info.codeSize = sizeof(GPU_TEST_MS_DEPTH_SPV); - module_info.pCode = GPU_TEST_MS_DEPTH_SPV; - vk::ShaderModule ms_depth_module = nullptr; - RequireVk(name, "MS depth content", - m_device.createShaderModule(&module_info, nullptr, &ms_depth_module), - "vkCreateShaderModule"); - std::array observer_bindings {}; - observer_bindings[0].binding = 0; - observer_bindings[0].descriptorType = vk::DescriptorType::eSampledImage; - observer_bindings[0].descriptorCount = 1; - observer_bindings[0].stageFlags = vk::ShaderStageFlagBits::eCompute; - observer_bindings[1].binding = 1; - observer_bindings[1].descriptorType = vk::DescriptorType::eStorageBuffer; - observer_bindings[1].descriptorCount = 1; - observer_bindings[1].stageFlags = vk::ShaderStageFlagBits::eCompute; - vk::DescriptorSetLayoutCreateInfo observer_layout_info {}; - observer_layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; - observer_layout_info.bindingCount = static_cast(observer_bindings.size()); - observer_layout_info.pBindings = observer_bindings.data(); - vk::DescriptorSetLayout observer_set_layout = nullptr; - RequireVk(name, "MS depth content", - m_device.createDescriptorSetLayout(&observer_layout_info, nullptr, - &observer_set_layout), - "vkCreateDescriptorSetLayout"); - const vk::PushConstantRange observer_push {vk::ShaderStageFlagBits::eCompute, 0, - sizeof(uint32_t)}; - vk::PipelineLayoutCreateInfo observer_pipeline_layout_info {}; - observer_pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; - observer_pipeline_layout_info.setLayoutCount = 1; - observer_pipeline_layout_info.pSetLayouts = &observer_set_layout; - observer_pipeline_layout_info.pushConstantRangeCount = 1; - observer_pipeline_layout_info.pPushConstantRanges = &observer_push; - vk::PipelineLayout observer_pipeline_layout = nullptr; - RequireVk(name, "MS depth content", - m_device.createPipelineLayout(&observer_pipeline_layout_info, nullptr, - &observer_pipeline_layout), - "vkCreatePipelineLayout"); - vk::PipelineShaderStageCreateInfo observer_stage {}; - observer_stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; - observer_stage.stage = vk::ShaderStageFlagBits::eCompute; - observer_stage.module = ms_depth_module; - observer_stage.pName = "main"; - vk::ComputePipelineCreateInfo observer_pipeline_info {}; - observer_pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo; - observer_pipeline_info.stage = observer_stage; - observer_pipeline_info.layout = observer_pipeline_layout; - vk::Pipeline observer_depth_pipeline = nullptr; - RequireVk(name, "MS depth content", - m_device.createComputePipelines(nullptr, 1, &observer_pipeline_info, nullptr, - &observer_depth_pipeline), - "vkCreateComputePipelines"); - const std::array observer_pool_sizes {{ - {vk::DescriptorType::eSampledImage, 2}, - {vk::DescriptorType::eStorageBuffer, 2}, - }}; - vk::DescriptorPoolCreateInfo observer_pool_info {}; - observer_pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; - observer_pool_info.maxSets = 2; - observer_pool_info.poolSizeCount = static_cast(observer_pool_sizes.size()); - observer_pool_info.pPoolSizes = observer_pool_sizes.data(); - vk::DescriptorPool observer_pool = nullptr; - RequireVk(name, "MS depth content", - m_device.createDescriptorPool(&observer_pool_info, nullptr, &observer_pool), - "vkCreateDescriptorPool"); - const std::array observer_layouts {observer_set_layout, - observer_set_layout}; - std::array observer_sets {}; - vk::DescriptorSetAllocateInfo observer_allocate {}; - observer_allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo; - observer_allocate.descriptorPool = observer_pool; - observer_allocate.descriptorSetCount = static_cast(observer_sets.size()); - observer_allocate.pSetLayouts = observer_layouts.data(); - RequireVk(name, "MS depth content", - m_device.allocateDescriptorSets(&observer_allocate, observer_sets.data()), - "vkAllocateDescriptorSets"); - std::array ms_observer_outputs { - CreateHostBuffer(name, 4 * sizeof(u32), vk::BufferUsageFlagBits::eStorageBuffer, - std::vector(4, 0)), - CreateHostBuffer(name, 4 * sizeof(u32), vk::BufferUsageFlagBits::eStorageBuffer, - std::vector(4, 0))}; - const std::array observed_images {ms_depth_image, ms_depth_image_4x}; - const std::array observed_samples {2, 4}; - for (uint32_t index = 0; index < observed_images.size(); index++) { - auto& observed = texture_cache.GetImage(observed_images[index]); - observed.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, {}, command.Handle()); - ImageViewInfo view_info {}; - view_info.format = observed.backing.format; - view_info.type = vk::ImageViewType::e2D; - view_info.aspect = vk::ImageAspectFlagBits::eDepth; - view_info.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR, - vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR}; - view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto view = observed.FindView(view_info); - const vk::DescriptorImageInfo image_info {nullptr, view, - vk::ImageLayout::eShaderReadOnlyOptimal}; - const vk::DescriptorBufferInfo buffer_info {ms_observer_outputs[index].buffer, 0, - ms_observer_outputs[index].size}; - std::array writes {}; - writes[0].sType = vk::StructureType::eWriteDescriptorSet; - writes[0].dstSet = observer_sets[index]; - writes[0].dstBinding = 0; - writes[0].descriptorCount = 1; - writes[0].descriptorType = vk::DescriptorType::eSampledImage; - writes[0].pImageInfo = &image_info; - writes[1].sType = vk::StructureType::eWriteDescriptorSet; - writes[1].dstSet = observer_sets[index]; - writes[1].dstBinding = 1; - writes[1].descriptorCount = 1; - writes[1].descriptorType = vk::DescriptorType::eStorageBuffer; - writes[1].pBufferInfo = &buffer_info; - m_device.updateDescriptorSets(static_cast(writes.size()), writes.data(), - 0, nullptr); - command.Handle().bindPipeline(vk::PipelineBindPoint::eCompute, - observer_depth_pipeline); - command.Handle().bindDescriptorSets(vk::PipelineBindPoint::eCompute, - observer_pipeline_layout, 0, 1, - &observer_sets[index], 0, nullptr); - command.Handle().pushConstants( - observer_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, - sizeof(observed_samples[index]), &observed_samples[index]); - command.Handle().dispatch(1, 1, 1); - HostReadBarrier(ms_observer_outputs[index].buffer, ms_observer_outputs[index].size, - vk::PipelineStageFlagBits::eComputeShader, - vk::AccessFlagBits::eShaderWrite); - } + vk::ShaderModuleCreateInfo module_info{}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = sizeof(GPU_TEST_MS_DEPTH_SPV); + module_info.pCode = GPU_TEST_MS_DEPTH_SPV; + vk::ShaderModule ms_depth_module = nullptr; + RequireVk( + name, "MS depth content", + m_device.createShaderModule(&module_info, nullptr, &ms_depth_module), + "vkCreateShaderModule"); + std::array observer_bindings{}; + observer_bindings[0].binding = 0; + observer_bindings[0].descriptorType = vk::DescriptorType::eSampledImage; + observer_bindings[0].descriptorCount = 1; + observer_bindings[0].stageFlags = vk::ShaderStageFlagBits::eCompute; + observer_bindings[1].binding = 1; + observer_bindings[1].descriptorType = vk::DescriptorType::eStorageBuffer; + observer_bindings[1].descriptorCount = 1; + observer_bindings[1].stageFlags = vk::ShaderStageFlagBits::eCompute; + vk::DescriptorSetLayoutCreateInfo observer_layout_info{}; + observer_layout_info.sType = + vk::StructureType::eDescriptorSetLayoutCreateInfo; + observer_layout_info.bindingCount = + static_cast(observer_bindings.size()); + observer_layout_info.pBindings = observer_bindings.data(); + vk::DescriptorSetLayout observer_set_layout = nullptr; + RequireVk(name, "MS depth content", + m_device.createDescriptorSetLayout( + &observer_layout_info, nullptr, &observer_set_layout), + "vkCreateDescriptorSetLayout"); + const vk::PushConstantRange observer_push{ + vk::ShaderStageFlagBits::eCompute, 0, sizeof(uint32_t)}; + vk::PipelineLayoutCreateInfo observer_pipeline_layout_info{}; + observer_pipeline_layout_info.sType = + vk::StructureType::ePipelineLayoutCreateInfo; + observer_pipeline_layout_info.setLayoutCount = 1; + observer_pipeline_layout_info.pSetLayouts = &observer_set_layout; + observer_pipeline_layout_info.pushConstantRangeCount = 1; + observer_pipeline_layout_info.pPushConstantRanges = &observer_push; + vk::PipelineLayout observer_pipeline_layout = nullptr; + RequireVk(name, "MS depth content", + m_device.createPipelineLayout(&observer_pipeline_layout_info, + nullptr, + &observer_pipeline_layout), + "vkCreatePipelineLayout"); + vk::PipelineShaderStageCreateInfo observer_stage{}; + observer_stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + observer_stage.stage = vk::ShaderStageFlagBits::eCompute; + observer_stage.module = ms_depth_module; + observer_stage.pName = "main"; + vk::ComputePipelineCreateInfo observer_pipeline_info{}; + observer_pipeline_info.sType = + vk::StructureType::eComputePipelineCreateInfo; + observer_pipeline_info.stage = observer_stage; + observer_pipeline_info.layout = observer_pipeline_layout; + vk::Pipeline observer_depth_pipeline = nullptr; + RequireVk( + name, "MS depth content", + m_device.createComputePipelines(nullptr, 1, &observer_pipeline_info, + nullptr, &observer_depth_pipeline), + "vkCreateComputePipelines"); + const std::array observer_pool_sizes{{ + {vk::DescriptorType::eSampledImage, 2}, + {vk::DescriptorType::eStorageBuffer, 2}, + }}; + vk::DescriptorPoolCreateInfo observer_pool_info{}; + observer_pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; + observer_pool_info.maxSets = 2; + observer_pool_info.poolSizeCount = + static_cast(observer_pool_sizes.size()); + observer_pool_info.pPoolSizes = observer_pool_sizes.data(); + vk::DescriptorPool observer_pool = nullptr; + RequireVk(name, "MS depth content", + m_device.createDescriptorPool(&observer_pool_info, nullptr, + &observer_pool), + "vkCreateDescriptorPool"); + const std::array observer_layouts{ + observer_set_layout, observer_set_layout}; + std::array observer_sets{}; + vk::DescriptorSetAllocateInfo observer_allocate{}; + observer_allocate.sType = vk::StructureType::eDescriptorSetAllocateInfo; + observer_allocate.descriptorPool = observer_pool; + observer_allocate.descriptorSetCount = + static_cast(observer_sets.size()); + observer_allocate.pSetLayouts = observer_layouts.data(); + RequireVk(name, "MS depth content", + m_device.allocateDescriptorSets(&observer_allocate, + observer_sets.data()), + "vkAllocateDescriptorSets"); + std::array ms_observer_outputs{ + CreateHostBuffer(name, 4 * sizeof(u32), + vk::BufferUsageFlagBits::eStorageBuffer, + std::vector(4, 0)), + CreateHostBuffer(name, 4 * sizeof(u32), + vk::BufferUsageFlagBits::eStorageBuffer, + std::vector(4, 0))}; + const std::array observed_images{ms_depth_image, + ms_depth_image_4x}; + const std::array observed_samples{2, 4}; + for (uint32_t index = 0; index < observed_images.size(); index++) { + auto &observed = texture_cache.GetImage(observed_images[index]); + observed.Transit(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, {}, + command.Handle()); + ImageViewInfo view_info{}; + view_info.format = observed.backing.format; + view_info.type = vk::ImageViewType::e2D; + view_info.aspect = vk::ImageAspectFlagBits::eDepth; + view_info.mapping = {vk::ComponentSwizzle::eR, vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR, + vk::ComponentSwizzle::eR}; + view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto view = observed.FindView(view_info); + const vk::DescriptorImageInfo image_info{ + nullptr, view, vk::ImageLayout::eShaderReadOnlyOptimal}; + const vk::DescriptorBufferInfo buffer_info{ + ms_observer_outputs[index].buffer, 0, + ms_observer_outputs[index].size}; + std::array writes{}; + writes[0].sType = vk::StructureType::eWriteDescriptorSet; + writes[0].dstSet = observer_sets[index]; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = vk::DescriptorType::eSampledImage; + writes[0].pImageInfo = &image_info; + writes[1].sType = vk::StructureType::eWriteDescriptorSet; + writes[1].dstSet = observer_sets[index]; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = vk::DescriptorType::eStorageBuffer; + writes[1].pBufferInfo = &buffer_info; + m_device.updateDescriptorSets(static_cast(writes.size()), + writes.data(), 0, nullptr); + command.Handle().bindPipeline(vk::PipelineBindPoint::eCompute, + observer_depth_pipeline); + command.Handle().bindDescriptorSets(vk::PipelineBindPoint::eCompute, + observer_pipeline_layout, 0, 1, + &observer_sets[index], 0, nullptr); + command.Handle().pushConstants( + observer_pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, + sizeof(observed_samples[index]), &observed_samples[index]); + command.Handle().dispatch(1, 1, 1); + HostReadBarrier(ms_observer_outputs[index].buffer, + ms_observer_outputs[index].size, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eShaderWrite); + } - scheduler.Finish(); + scheduler.Finish(); - const auto mip_prefix_words = - ReadBuffer(name, mip_prefix_readback, mip_prefix_size / sizeof(uint32_t)); - const auto mip_formatted_words = - ReadBuffer(name, mip_formatted_readback, mip_guest_size / sizeof(uint32_t)); - const auto mip0_word = mip_sizes[0].offset / sizeof(uint32_t); - Require(name, "formatted mip prefix content", - mip0_word < mip_prefix_words.size() && mip0_word < mip_native.size() && - mip_prefix_words[mip0_word] == mip_native[mip0_word], - "fitting mip-prefix synchronization copied " - "stale guest backing"); - Require(name, "formatted Buffer content", - mip0_word < mip_formatted_words.size() && mip0_word < mip_native.size() && - mip_formatted_words[mip0_word] == mip_native[mip0_word], - "formatted Buffer read bypassed authoritative " - "native image mip data"); - Require(name, "CPU-dirty formatted mirror content", - ReadBuffer(name, mirror_cpu_readback, 1) == std::vector {mirror_cpu_value}, - "formatted Buffer mirror published stale " - "native image bytes"); - Require(name, "Buffer-superseded exact content", - ReadBuffer(name, exact_buffer_readback, 1) == - std::vector {exact_buffer_value}, - "exact-format recreation initialized from " - "stale guest bytes"); - Require(name, "partial-page CPU refresh content", - ReadBuffer(name, partial_cpu_refresh_readback, 1) == - std::vector {partial_cpu_refresh_value}, - "cached buffer uploaded bytes outside the exact " - "staged guest range"); - const auto mixed_source_words = - ReadBuffer(name, mixed_source_readback, mixed_source_size / sizeof(u32)); - Require(name, "mixed-page image content", - mixed_source_words.front() == mixed_cpu_value && - mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value, - "mixed CPU/GPU source upload lost one " - "ownership domain"); - Require(name, "BGRA16 content", - ReadBuffer(name, bgra16_readback, 2) == - std::vector {0x40004200u, 0x44003c00u}, - "GPU BGRA16 swap did not publish RGBA half-word order"); - scheduler.Finish(); - const auto layered_words = ReadBuffer(name, layered_readback, layered_values.size()); - const std::array layered_expected { - layered_values[0], layered_values[1], layered_values[2], layered_values[3], - layered_values[4], layered_values[5], layered_values[6], layered_values[7], - layered_values[8], layered_values[9]}; - bool layered_content = layered_words.size() == layered_expected.size(); - for (uint32_t index = 0; layered_content && index < layered_expected.size(); index++) { - layered_content &= - layered_words[index] == std::bit_cast(layered_expected[index]); - } - Require(name, "layered mipped depth content", layered_content, - "depth/color conversion changed a mip or array-layer value"); - const std::array, 2> expected_ms_depth {{ - {0x2000 / 65535.0f, 0xe000 / 65535.0f, 0.0f, 0.0f}, - {0.0f, 0x4000 / 65535.0f, 0x8000 / 65535.0f, 1.0f}, - }}; - for (uint32_t image = 0; image < expected_ms_depth.size(); image++) { - const auto words = ReadBuffer(name, ms_observer_outputs[image], 4); - bool content = words.size() == 4; - for (uint32_t sample = 0; content && sample < observed_samples[image]; sample++) { - const float actual = std::bit_cast(words[sample]); - content &= - std::abs(actual - expected_ms_depth[image][sample]) <= 1.5f / 65535.0f; - } - Require(name, image == 0 ? "two-sample depth content" : "four-sample depth content", - content, "color-to-multisample-depth changed a packed sample value"); - } + const auto mip_prefix_words = ReadBuffer( + name, mip_prefix_readback, mip_prefix_size / sizeof(uint32_t)); + const auto mip_formatted_words = ReadBuffer( + name, mip_formatted_readback, mip_guest_size / sizeof(uint32_t)); + const auto mip0_word = mip_sizes[0].offset / sizeof(uint32_t); + Require(name, "formatted mip prefix content", + mip0_word < mip_prefix_words.size() && + mip0_word < mip_native.size() && + mip_prefix_words[mip0_word] == mip_native[mip0_word], + "fitting mip-prefix synchronization copied " + "stale guest backing"); + Require(name, "formatted Buffer content", + mip0_word < mip_formatted_words.size() && + mip0_word < mip_native.size() && + mip_formatted_words[mip0_word] == mip_native[mip0_word], + "formatted Buffer read bypassed authoritative " + "native image mip data"); + Require(name, "CPU-dirty formatted mirror content", + ReadBuffer(name, mirror_cpu_readback, 1) == + std::vector{mirror_cpu_value}, + "formatted Buffer mirror published stale " + "native image bytes"); + Require(name, "Buffer-superseded exact content", + ReadBuffer(name, exact_buffer_readback, 1) == + std::vector{exact_buffer_value}, + "exact-format recreation initialized from " + "stale guest bytes"); + Require(name, "partial-page CPU refresh content", + ReadBuffer(name, partial_cpu_refresh_readback, 1) == + std::vector{partial_cpu_refresh_value}, + "cached buffer uploaded bytes outside the exact " + "staged guest range"); + const auto mixed_source_words = ReadBuffer( + name, mixed_source_readback, mixed_source_size / sizeof(u32)); + Require(name, "mixed-page image content", + mixed_source_words.front() == mixed_cpu_value && + mixed_source_words[0x1000 / sizeof(u32)] == mixed_gpu_value, + "mixed CPU/GPU source upload lost one " + "ownership domain"); + Require(name, "BGRA16 content", + ReadBuffer(name, bgra16_readback, 2) == + std::vector{0x40004200u, 0x44003c00u}, + "GPU BGRA16 swap did not publish RGBA half-word order"); + scheduler.Finish(); + const auto layered_words = + ReadBuffer(name, layered_readback, layered_values.size()); + const std::array layered_expected{ + layered_values[0], layered_values[1], layered_values[2], + layered_values[3], layered_values[4], layered_values[5], + layered_values[6], layered_values[7], layered_values[8], + layered_values[9]}; + bool layered_content = layered_words.size() == layered_expected.size(); + for (uint32_t index = 0; + layered_content && index < layered_expected.size(); index++) { + layered_content &= + layered_words[index] == std::bit_cast(layered_expected[index]); + } + Require(name, "layered mipped depth content", layered_content, + "depth/color conversion changed a mip or array-layer value"); + const std::array, 2> expected_ms_depth{{ + {0x2000 / 65535.0f, 0xe000 / 65535.0f, 0.0f, 0.0f}, + {0.0f, 0x4000 / 65535.0f, 0x8000 / 65535.0f, 1.0f}, + }}; + for (uint32_t image = 0; image < expected_ms_depth.size(); image++) { + const auto words = ReadBuffer(name, ms_observer_outputs[image], 4); + bool content = words.size() == 4; + for (uint32_t sample = 0; content && sample < observed_samples[image]; + sample++) { + const float actual = std::bit_cast(words[sample]); + content &= std::abs(actual - expected_ms_depth[image][sample]) <= + 1.5f / 65535.0f; + } + Require(name, + image == 0 ? "two-sample depth content" + : "four-sample depth content", + content, + "color-to-multisample-depth changed a packed sample value"); + } - auto unrelated_stencil_alias = MakeLinearDesc( - base + second_stencil_offset, ms_stencil_size, vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, - {static_cast(ms_stencil_size / sizeof(uint32_t)), 1, 1}, 1, 4, 1); - const auto unrelated_stencil_image = texture_cache.FindImage(unrelated_stencil_alias); - Require(name, "stencil association filtering", - unrelated_stencil_image && unrelated_stencil_image != ms_depth_image && - !texture_cache.GetImage(unrelated_stencil_image).depth_id, - "an unrelated color alias was hijacked by the stencil " - "association record"); + auto unrelated_stencil_alias = MakeLinearDesc( + base + second_stencil_offset, ms_stencil_size, vk::Format::eR32Uint, + Prospero::BufferFormat::k32UInt, Prospero::ImageType::kColor2D, + {static_cast(ms_stencil_size / sizeof(uint32_t)), 1, 1}, 1, + 4, 1); + const auto unrelated_stencil_image = + texture_cache.FindImage(unrelated_stencil_alias); + Require(name, "stencil association filtering", + unrelated_stencil_image && + unrelated_stencil_image != ms_depth_image && + !texture_cache.GetImage(unrelated_stencil_image).depth_id, + "an unrelated color alias was hijacked by the stencil " + "association record"); - texture_cache.UnmapMemory(base + second_stencil_offset, ms_stencil_size); - Require(name, "stencil proxy unmap", - !texture_cache.FindImageFromRange(base + second_stencil_offset, ms_stencil_size, - false) && - texture_cache.GetImage(ms_depth_image).backing.image == ms_depth_backing, - "stencil unmap removed the depth backing or retained its proxy"); - const auto reassociated_depth = texture_cache.FindImage(switched_ms_depth); - TextureCacheTestAccess::AssociateStencil(texture_cache, reassociated_depth, - switched_ms_depth.info.stencil); - const auto reassociated_proxy = texture_cache.FindImageFromRange( - base + second_stencil_offset, ms_stencil_size, false); - Require(name, "stencil proxy reassociation", - reassociated_depth == ms_depth_image && reassociated_proxy && - texture_cache.GetImage(reassociated_proxy).depth_id == ms_depth_image, - "stencil proxy did not re-associate with the surviving depth"); - texture_cache.UnmapMemory(ms_depth_desc.info.data.address, - ms_depth_desc.info.data.size); - Require(name, "depth unmap proxy retirement", - !texture_cache.FindImageFromRange(base + second_stencil_offset, ms_stencil_size, - false), - "unmapping depth retained its re-associated stencil proxy"); + texture_cache.UnmapMemory(base + second_stencil_offset, ms_stencil_size); + Require(name, "stencil proxy unmap", + !texture_cache.FindImageFromRange(base + second_stencil_offset, + ms_stencil_size, false) && + texture_cache.GetImage(ms_depth_image).backing.image == + ms_depth_backing, + "stencil unmap removed the depth backing or retained its proxy"); + const auto reassociated_depth = + texture_cache.FindImage(switched_ms_depth); + TextureCacheTestAccess::AssociateStencil( + texture_cache, reassociated_depth, switched_ms_depth.info.stencil); + const auto reassociated_proxy = texture_cache.FindImageFromRange( + base + second_stencil_offset, ms_stencil_size, false); + Require(name, "stencil proxy reassociation", + reassociated_depth == ms_depth_image && reassociated_proxy && + texture_cache.GetImage(reassociated_proxy).depth_id == + ms_depth_image, + "stencil proxy did not re-associate with the surviving depth"); + texture_cache.UnmapMemory(ms_depth_desc.info.data.address, + ms_depth_desc.info.data.size); + Require(name, "depth unmap proxy retirement", + !texture_cache.FindImageFromRange(base + second_stencil_offset, + ms_stencil_size, false), + "unmapping depth retained its re-associated stencil proxy"); - constexpr uint64_t exact_image_offset = 0x334100; - constexpr uint64_t dirty_sibling_offset = 0x334200; - constexpr uint32_t dirty_sibling_value = 0xc001d00du; - auto dirty_sibling = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), base + dirty_sibling_offset, sizeof(dirty_sibling_value), true, - false); - Require(name, "same-page dirty sibling allocation", dirty_sibling.owner != nullptr, - "failed to create the disjoint same-page Buffer owner"); - scheduler.Current().RetainResourceUntilFence(dirty_sibling.owner); - resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, - sizeof(dirty_sibling_value), dirty_sibling_value); - auto exact_image_desc = - MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - const auto exact_image = texture_cache.FindImage(exact_image_desc); - (void)texture_cache.FindTexture(exact_image, exact_image_desc); - texture_cache.MarkGpuWritten(exact_image); - Require(name, "same-page exact alias ownership", - resources.GetBufferCache().HasGpuDirtyBytes(base + dirty_sibling_offset, - sizeof(dirty_sibling_value)) && - !resources.GetBufferCache().HasGpuDirtyBytes(base + exact_image_offset, - sizeof(uint32_t)) && - texture_cache.GetImage(exact_image).IsGpuModified(), - "image ownership discarded or conflicted with disjoint dirty " - "Buffer bytes on the same tracker page"); - resources.GetBufferCache().ReadMemory(base + dirty_sibling_offset, - sizeof(dirty_sibling_value)); + constexpr uint64_t exact_image_offset = 0x334100; + constexpr uint64_t dirty_sibling_offset = 0x334200; + constexpr uint32_t dirty_sibling_value = 0xc001d00du; + auto dirty_sibling = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), base + dirty_sibling_offset, + sizeof(dirty_sibling_value), true, false); + Require(name, "same-page dirty sibling allocation", + dirty_sibling.owner != nullptr, + "failed to create the disjoint same-page Buffer owner"); + scheduler.Current().RetainResourceUntilFence(dirty_sibling.owner); + resources.GetBufferCache().FillBuffer(base + dirty_sibling_offset, + sizeof(dirty_sibling_value), + dirty_sibling_value); + auto exact_image_desc = + MakeLinearDesc(base + exact_image_offset, sizeof(uint32_t), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + const auto exact_image = texture_cache.FindImage(exact_image_desc); + (void)texture_cache.FindTexture(exact_image, exact_image_desc); + texture_cache.MarkGpuWritten(exact_image); + Require(name, "same-page exact alias ownership", + resources.GetBufferCache().HasGpuDirtyBytes( + base + dirty_sibling_offset, sizeof(dirty_sibling_value)) && + !resources.GetBufferCache().HasGpuDirtyBytes( + base + exact_image_offset, sizeof(uint32_t)) && + texture_cache.GetImage(exact_image).IsGpuModified(), + "image ownership discarded or conflicted with disjoint dirty " + "Buffer bytes on the same tracker page"); + resources.GetBufferCache().ReadMemory(base + dirty_sibling_offset, + sizeof(dirty_sibling_value)); - constexpr std::array gc_image_offsets {0x330000, 0x332000}; - constexpr std::array gc_image_values {0x76543210u, 0x89abcdefu}; - constexpr std::array gc_stale_values {0x10293847u, 0x56473829u}; - for (size_t index = 0; index < gc_image_offsets.size(); index++) { - std::memcpy(memory + gc_image_offsets[index], &gc_image_values[index], - sizeof(uint32_t)); - } - auto gc_image_desc_a = - MakeLinearDesc(base + gc_image_offsets[0], sizeof(uint32_t), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - auto gc_image_desc_b = gc_image_desc_a; - gc_image_desc_b.info.data.address = base + gc_image_offsets[1]; - auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size, true, false); - Require(name, "clean pre-image buffer alias", - clean_buffer_alias.owner != nullptr && clean_buffer_alias.buffer != nullptr, - "failed to create the clean cached Buffer alias"); - scheduler.Current().RetainResourceUntilFence(clean_buffer_alias.owner); - resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size); - const std::array gc_images {texture_cache.FindImage(gc_image_desc_a), - texture_cache.FindImage(gc_image_desc_b)}; - Require(name, "non-GPU image range validity", - !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size), - "FindImageFromRange accepted an image without current GPU " - "contents"); - (void)texture_cache.FindTexture(gc_images[0], gc_image_desc_a); - (void)texture_cache.FindTexture(gc_images[1], gc_image_desc_b); - for (const auto image: gc_images) { - texture_cache.MarkGpuWritten(image); - } - Require(name, "GPU image range validity", - texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size) == - gc_images[0], - "FindImageFromRange rejected a clean GPU-current image"); - texture_cache.GetImage(gc_images[0]) - .InvalidateCpuWrite(gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size); - Require(name, "CPU-dirty image range validity", - !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size), - "FindImageFromRange accepted CPU-dirty native contents"); - texture_cache.MarkGpuWritten(gc_images[0]); - for (size_t index = 0; index < gc_image_offsets.size(); index++) { - Libs::LibKernel::Memory::WriteBacking(base + gc_image_offsets[index], - &gc_stale_values[index], sizeof(uint32_t)); - } - TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache, gc_images, 17, - UINT64_MAX); - texture_cache.RunGarbageCollector(); - Require(name, "downloadable image pre-pressure retention", - std::ranges::all_of(gc_images, - [&](ImageId image) { - return TextureCacheTestAccess::Contains(texture_cache, - image); - }), - "GC retired safely downloadable GPU images before pressure"); - const auto gc_batch_tick = scheduler.CurrentTick(); - TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache, gc_images, 81, 0); - texture_cache.RunGarbageCollector(); - std::array gc_before_completion {}; - for (size_t index = 0; index < gc_image_offsets.size(); index++) { - Libs::LibKernel::Memory::TryReadBacking( - base + gc_image_offsets[index], &gc_before_completion[index], sizeof(uint32_t)); - } - Require(name, "batched image pressure retirement", - std::ranges::none_of(gc_images, - [&](ImageId image) { - return TextureCacheTestAccess::Contains(texture_cache, - image); - }) && - scheduler.CurrentTick() == gc_batch_tick && - gc_before_completion == gc_stale_values, - "GC submitted per image or published a readback before GPU " - "completion"); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), gc_image_desc_a.info.data.address, - gc_image_desc_a.info.data.size, false, true); - Require(name, "post-publication Buffer reacquire", - refreshed_buffer_alias.buffer != nullptr, - "Buffer lookup failed after the retired image publication"); - if (refreshed_buffer_alias.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(refreshed_buffer_alias.owner); - } - std::array gc_after_completion {}; - for (size_t index = 0; index < gc_image_offsets.size(); index++) { - Libs::LibKernel::Memory::TryReadBacking( - base + gc_image_offsets[index], &gc_after_completion[index], sizeof(uint32_t)); - } - Require(name, "batched image readback publication", - scheduler.CurrentTick() == gc_batch_tick + 1 && - gc_after_completion == gc_image_values, - "one submission did not publish both deferred image readbacks"); - Require(name, "refreshed post-image buffer alias", - refreshed_buffer_alias.buffer != nullptr, - "failed to reacquire the cached Buffer alias after image " - "publication"); - if (refreshed_buffer_alias.owner != nullptr) { - scheduler.Current().RetainResourceUntilFence(refreshed_buffer_alias.owner); - } - auto alias_readback = CreateHostBuffer(name, sizeof(uint32_t), - vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy alias_copy {refreshed_buffer_alias.offset, 0, sizeof(uint32_t)}; - scheduler.Current().Handle().copyBuffer(refreshed_buffer_alias.buffer, - alias_readback.buffer, 1, &alias_copy); - HostReadBarrier(alias_readback.buffer, alias_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - scheduler.FinishCurrent(); - Require(name, "post-image Buffer alias content", - ReadBuffer(name, alias_readback, 1) == std::vector {gc_image_values[0]}, - "a clean cached Buffer alias survived image ownership with " - "stale native bytes"); - DestroyBuffer(&alias_readback); + constexpr std::array gc_image_offsets{0x330000, 0x332000}; + constexpr std::array gc_image_values{0x76543210u, + 0x89abcdefu}; + constexpr std::array gc_stale_values{0x10293847u, + 0x56473829u}; + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + std::memcpy(memory + gc_image_offsets[index], &gc_image_values[index], + sizeof(uint32_t)); + } + auto gc_image_desc_a = + MakeLinearDesc(base + gc_image_offsets[0], sizeof(uint32_t), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + auto gc_image_desc_b = gc_image_desc_a; + gc_image_desc_b.info.data.address = base + gc_image_offsets[1]; + auto clean_buffer_alias = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size, true, false); + Require(name, "clean pre-image buffer alias", + clean_buffer_alias.owner != nullptr && + clean_buffer_alias.buffer != nullptr, + "failed to create the clean cached Buffer alias"); + scheduler.Current().RetainResourceUntilFence(clean_buffer_alias.owner); + resources.GetBufferCache().ReadMemory(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size); + const std::array gc_images{texture_cache.FindImage(gc_image_desc_a), + texture_cache.FindImage(gc_image_desc_b)}; + Require( + name, "non-GPU image range validity", + !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size), + "FindImageFromRange accepted an image without current GPU " + "contents"); + (void)texture_cache.FindTexture(gc_images[0], gc_image_desc_a); + (void)texture_cache.FindTexture(gc_images[1], gc_image_desc_b); + for (const auto image : gc_images) { + texture_cache.MarkGpuWritten(image); + } + Require(name, "GPU image range validity", + texture_cache.FindImageFromRange( + gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size) == gc_images[0], + "FindImageFromRange rejected a clean GPU-current image"); + texture_cache.GetImage(gc_images[0]) + .InvalidateCpuWrite(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size); + Require( + name, "CPU-dirty image range validity", + !texture_cache.FindImageFromRange(gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size), + "FindImageFromRange accepted CPU-dirty native contents"); + texture_cache.MarkGpuWritten(gc_images[0]); + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + Libs::LibKernel::Memory::WriteBacking(base + gc_image_offsets[index], + &gc_stale_values[index], + sizeof(uint32_t)); + } + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, gc_images, 17, UINT64_MAX); + texture_cache.RunGarbageCollector(); + Require(name, "downloadable image pre-pressure retention", + std::ranges::all_of(gc_images, + [&](ImageId image) { + return TextureCacheTestAccess::Contains( + texture_cache, image); + }), + "GC retired safely downloadable GPU images before pressure"); + const auto gc_batch_tick = scheduler.CurrentTick(); + TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache, + gc_images, 81, 0); + texture_cache.RunGarbageCollector(); + std::array gc_before_completion{}; + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index], + &gc_before_completion[index], + sizeof(uint32_t)); + } + Require(name, "batched image pressure retirement", + std::ranges::none_of(gc_images, + [&](ImageId image) { + return TextureCacheTestAccess::Contains( + texture_cache, image); + }) && + scheduler.CurrentTick() == gc_batch_tick && + gc_before_completion == gc_stale_values, + "GC submitted per image or published a readback before GPU " + "completion"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + auto refreshed_buffer_alias = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), gc_image_desc_a.info.data.address, + gc_image_desc_a.info.data.size, false, true); + Require(name, "post-publication Buffer reacquire", + refreshed_buffer_alias.buffer != nullptr, + "Buffer lookup failed after the retired image publication"); + if (refreshed_buffer_alias.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence( + refreshed_buffer_alias.owner); + } + std::array gc_after_completion{}; + for (size_t index = 0; index < gc_image_offsets.size(); index++) { + Libs::LibKernel::Memory::TryReadBacking(base + gc_image_offsets[index], + &gc_after_completion[index], + sizeof(uint32_t)); + } + Require(name, "batched image readback publication", + scheduler.CurrentTick() == gc_batch_tick + 1 && + gc_after_completion == gc_image_values, + "one submission did not publish both deferred image readbacks"); + Require(name, "refreshed post-image buffer alias", + refreshed_buffer_alias.buffer != nullptr, + "failed to reacquire the cached Buffer alias after image " + "publication"); + if (refreshed_buffer_alias.owner != nullptr) { + scheduler.Current().RetainResourceUntilFence( + refreshed_buffer_alias.owner); + } + auto alias_readback = CreateHostBuffer( + name, sizeof(uint32_t), vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy alias_copy{refreshed_buffer_alias.offset, 0, + sizeof(uint32_t)}; + scheduler.Current().Handle().copyBuffer( + refreshed_buffer_alias.buffer, alias_readback.buffer, 1, &alias_copy); + HostReadBarrier(alias_readback.buffer, alias_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + Require(name, "post-image Buffer alias content", + ReadBuffer(name, alias_readback, 1) == + std::vector{gc_image_values[0]}, + "a clean cached Buffer alias survived image ownership with " + "stale native bytes"); + DestroyBuffer(&alias_readback); - constexpr uint64_t submit_readback_offset = 0x336000; - constexpr uint32_t submit_readback_value = 0x13579bdfu; - constexpr uint32_t submit_readback_stale = 0x2468ace0u; - auto submit_readback_desc = MakeLinearDesc( - base + submit_readback_offset, sizeof(uint32_t), vk::Format::eR32Uint, - Prospero::BufferFormat::k32UInt, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - submit_readback_desc.type = BindingType::Storage; - submit_readback_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; - TextureCacheTestAccess::SetLinearReadback(texture_cache, true); - const auto submit_readback_image = texture_cache.FindImage(submit_readback_desc); - Require(name, "linear submit image clear", - texture_cache.ClearImageFromBuffer( - command, submit_readback_desc.info.data.address, - submit_readback_desc.info.data.size, submit_readback_value), - "failed to create GPU-current linear storage contents"); - TextureCacheTestAccess::TrackDownload(texture_cache, submit_readback_image); - Libs::LibKernel::Memory::WriteBacking(submit_readback_desc.info.data.address, - &submit_readback_stale, - sizeof(submit_readback_stale)); - const auto submit_readback_tick = scheduler.CurrentTick(); - texture_cache.ProcessDownloadImages(); - TextureCacheTestAccess::SetLinearReadback(texture_cache, false); - uint32_t submit_before_completion = 0; - Libs::LibKernel::Memory::TryReadBacking(submit_readback_desc.info.data.address, - &submit_before_completion, - sizeof(submit_before_completion)); - Require(name, "linear submit deferred publication", - submit_before_completion == submit_readback_stale && - scheduler.CurrentTick() == submit_readback_tick, - "submit-time linear readback published synchronously"); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - uint32_t submit_after_completion = 0; - Libs::LibKernel::Memory::TryReadBacking(submit_readback_desc.info.data.address, - &submit_after_completion, - sizeof(submit_after_completion)); - Require(name, "linear submit readback reuse", - submit_after_completion == submit_readback_value && - TextureCacheTestAccess::Contains(texture_cache, submit_readback_image) && - texture_cache.GetImage(submit_readback_image).IsGpuModified(), - "submit-time readback did not publish or retained no reusable " - "GPU-current image"); + constexpr uint64_t submit_readback_offset = 0x336000; + constexpr uint32_t submit_readback_value = 0x13579bdfu; + constexpr uint32_t submit_readback_stale = 0x2468ace0u; + auto submit_readback_desc = + MakeLinearDesc(base + submit_readback_offset, sizeof(uint32_t), + vk::Format::eR32Uint, Prospero::BufferFormat::k32UInt, + Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); + submit_readback_desc.type = BindingType::Storage; + submit_readback_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + TextureCacheTestAccess::SetLinearReadback(texture_cache, true); + const auto submit_readback_image = + texture_cache.FindImage(submit_readback_desc); + Require(name, "linear submit image clear", + texture_cache.ClearImageFromBuffer( + command, submit_readback_desc.info.data.address, + submit_readback_desc.info.data.size, submit_readback_value), + "failed to create GPU-current linear storage contents"); + TextureCacheTestAccess::TrackDownload(texture_cache, + submit_readback_image); + Libs::LibKernel::Memory::WriteBacking( + submit_readback_desc.info.data.address, &submit_readback_stale, + sizeof(submit_readback_stale)); + const auto submit_readback_tick = scheduler.CurrentTick(); + texture_cache.ProcessDownloadImages(); + TextureCacheTestAccess::SetLinearReadback(texture_cache, false); + uint32_t submit_before_completion = 0; + Libs::LibKernel::Memory::TryReadBacking( + submit_readback_desc.info.data.address, &submit_before_completion, + sizeof(submit_before_completion)); + Require(name, "linear submit deferred publication", + submit_before_completion == submit_readback_stale && + scheduler.CurrentTick() == submit_readback_tick, + "submit-time linear readback published synchronously"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + uint32_t submit_after_completion = 0; + Libs::LibKernel::Memory::TryReadBacking( + submit_readback_desc.info.data.address, &submit_after_completion, + sizeof(submit_after_completion)); + Require(name, "linear submit readback reuse", + submit_after_completion == submit_readback_value && + TextureCacheTestAccess::Contains(texture_cache, + submit_readback_image) && + texture_cache.GetImage(submit_readback_image).IsGpuModified(), + "submit-time readback did not publish or retained no reusable " + "GPU-current image"); - constexpr uint64_t linear_depth_offset = 0x33a000; - constexpr uint64_t linear_stencil_offset = 0x33b000; - constexpr uint32_t linear_depth_width = 3; - constexpr uint32_t linear_depth_height = 2; - constexpr uint32_t linear_depth_pitch = 4; - constexpr uint32_t linear_depth_layers = 2; - constexpr uint32_t linear_depth_words = - linear_depth_pitch * linear_depth_height * linear_depth_layers; - constexpr uint32_t linear_depth_clear = 0x3f400000u; - std::array linear_depth_guest {}; - for (uint32_t index = 0; index < linear_depth_words; index++) { - linear_depth_guest[index] = 0x51000000u + index; - } - constexpr std::array linear_stencil_guest {0x91, 0x82, 0x73, 0x64, - 0x55, 0x46, 0x37, 0x28}; - std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(), - sizeof(linear_depth_guest)); - std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(), - sizeof(linear_stencil_guest)); - auto linear_depth_desc = MakeLinearDesc( - base + linear_depth_offset, sizeof(linear_depth_guest), - vk::Format::eD32SfloatS8Uint, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {linear_depth_width, linear_depth_height, 1}, - linear_depth_layers, 4, 1); - linear_depth_desc.type = BindingType::DepthTarget; - linear_depth_desc.info.pitch = linear_depth_pitch; - linear_depth_desc.info.stencil = {base + linear_stencil_offset, - sizeof(linear_stencil_guest)}; - linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest), - linear_depth_pitch, linear_depth_height}; - linear_depth_desc.view_info.aspect = - vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; - linear_depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto linear_depth_image = texture_cache.FindImage(linear_depth_desc); - Require(name, "linear depth native clear", - texture_cache.ClearImageFromBuffer(command, linear_depth_desc.info.data.address, - linear_depth_desc.info.data.size, - linear_depth_clear), - "failed to create GPU-current padded linear depth contents"); - TextureCacheTestAccess::SetLinearReadback(texture_cache, true); - TextureCacheTestAccess::TrackDownload(texture_cache, linear_depth_image); - const auto linear_depth_tick = scheduler.CurrentTick(); - texture_cache.ProcessDownloadImages(); - TextureCacheTestAccess::SetLinearReadback(texture_cache, false); - std::array linear_depth_before {}; - std::memcpy(linear_depth_before.data(), memory + linear_depth_offset, - sizeof(linear_depth_before)); - Require(name, "linear depth deferred publication", - linear_depth_before == linear_depth_guest && - scheduler.CurrentTick() == linear_depth_tick, - "linear depth readback published before GPU completion"); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - std::array linear_depth_after {}; - std::array linear_stencil_after {}; - std::memcpy(linear_depth_after.data(), memory + linear_depth_offset, - sizeof(linear_depth_after)); - std::memcpy(linear_stencil_after.data(), memory + linear_stencil_offset, - sizeof(linear_stencil_after)); - bool linear_depth_matches = true; - uint32_t linear_depth_mismatch = UINT32_MAX; - for (uint32_t layer = 0; layer < linear_depth_layers; layer++) { - for (uint32_t y = 0; y < linear_depth_height; y++) { - for (uint32_t x = 0; x < linear_depth_pitch; x++) { - const auto index = layer * linear_depth_pitch * linear_depth_height + - y * linear_depth_pitch + x; - const auto expected = - x < linear_depth_width ? linear_depth_clear : linear_depth_guest[index]; - if (linear_depth_after[index] != expected && - linear_depth_mismatch == UINT32_MAX) { - linear_depth_mismatch = index; - } - linear_depth_matches &= linear_depth_after[index] == expected; - } - } - } - Require( - name, "linear depth native contents", - linear_depth_matches && linear_stencil_after == linear_stencil_guest && - TextureCacheTestAccess::Contains(texture_cache, linear_depth_image) && - texture_cache.GetImage(linear_depth_image).IsGpuModified(), - fmt::format("linear depth mismatch={} actual=0x{:08x} expected=0x{:08x} " - "stencil={} contains={} gpu={}", - linear_depth_mismatch, - linear_depth_mismatch == UINT32_MAX - ? 0 - : linear_depth_after[linear_depth_mismatch], - linear_depth_mismatch == UINT32_MAX - ? 0 - : (linear_depth_mismatch % linear_depth_pitch < linear_depth_width - ? linear_depth_clear - : linear_depth_guest[linear_depth_mismatch]), - linear_stencil_after == linear_stencil_guest, - TextureCacheTestAccess::Contains(texture_cache, linear_depth_image), - texture_cache.GetImage(linear_depth_image).IsGpuModified()) - .c_str()); + constexpr uint64_t linear_depth_offset = 0x33a000; + constexpr uint64_t linear_stencil_offset = 0x33b000; + constexpr uint32_t linear_depth_width = 3; + constexpr uint32_t linear_depth_height = 2; + constexpr uint32_t linear_depth_pitch = 4; + constexpr uint32_t linear_depth_layers = 2; + constexpr uint32_t linear_depth_words = + linear_depth_pitch * linear_depth_height * linear_depth_layers; + constexpr uint32_t linear_depth_clear = 0x3f400000u; + std::array linear_depth_guest{}; + for (uint32_t index = 0; index < linear_depth_words; index++) { + linear_depth_guest[index] = 0x51000000u + index; + } + constexpr std::array linear_stencil_guest{ + 0x91, 0x82, 0x73, 0x64, 0x55, 0x46, 0x37, 0x28}; + std::memcpy(memory + linear_depth_offset, linear_depth_guest.data(), + sizeof(linear_depth_guest)); + std::memcpy(memory + linear_stencil_offset, linear_stencil_guest.data(), + sizeof(linear_stencil_guest)); + auto linear_depth_desc = MakeLinearDesc( + base + linear_depth_offset, sizeof(linear_depth_guest), + vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float, + Prospero::ImageType::kColor2D, + {linear_depth_width, linear_depth_height, 1}, linear_depth_layers, 4, + 1); + linear_depth_desc.type = BindingType::DepthTarget; + linear_depth_desc.info.pitch = linear_depth_pitch; + linear_depth_desc.info.stencil = {base + linear_stencil_offset, + sizeof(linear_stencil_guest)}; + linear_depth_desc.info.mip_layout[0] = {0, sizeof(linear_depth_guest), + linear_depth_pitch, + linear_depth_height}; + linear_depth_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + linear_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto linear_depth_image = + texture_cache.FindImage(linear_depth_desc); + Require(name, "linear depth native clear", + texture_cache.ClearImageFromBuffer( + command, linear_depth_desc.info.data.address, + linear_depth_desc.info.data.size, linear_depth_clear), + "failed to create GPU-current padded linear depth contents"); + TextureCacheTestAccess::SetLinearReadback(texture_cache, true); + TextureCacheTestAccess::TrackDownload(texture_cache, linear_depth_image); + const auto linear_depth_tick = scheduler.CurrentTick(); + texture_cache.ProcessDownloadImages(); + TextureCacheTestAccess::SetLinearReadback(texture_cache, false); + std::array linear_depth_before{}; + std::memcpy(linear_depth_before.data(), memory + linear_depth_offset, + sizeof(linear_depth_before)); + Require(name, "linear depth deferred publication", + linear_depth_before == linear_depth_guest && + scheduler.CurrentTick() == linear_depth_tick, + "linear depth readback published before GPU completion"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::array linear_depth_after{}; + std::array linear_stencil_after{}; + std::memcpy(linear_depth_after.data(), memory + linear_depth_offset, + sizeof(linear_depth_after)); + std::memcpy(linear_stencil_after.data(), memory + linear_stencil_offset, + sizeof(linear_stencil_after)); + bool linear_depth_matches = true; + uint32_t linear_depth_mismatch = UINT32_MAX; + for (uint32_t layer = 0; layer < linear_depth_layers; layer++) { + for (uint32_t y = 0; y < linear_depth_height; y++) { + for (uint32_t x = 0; x < linear_depth_pitch; x++) { + const auto index = + layer * linear_depth_pitch * linear_depth_height + + y * linear_depth_pitch + x; + const auto expected = x < linear_depth_width + ? linear_depth_clear + : linear_depth_guest[index]; + if (linear_depth_after[index] != expected && + linear_depth_mismatch == UINT32_MAX) { + linear_depth_mismatch = index; + } + linear_depth_matches &= linear_depth_after[index] == expected; + } + } + } + Require(name, "linear depth native contents", + linear_depth_matches && + linear_stencil_after == linear_stencil_guest && + TextureCacheTestAccess::Contains(texture_cache, + linear_depth_image) && + texture_cache.GetImage(linear_depth_image).IsGpuModified(), + fmt::format( + "linear depth mismatch={} actual=0x{:08x} expected=0x{:08x} " + "stencil={} contains={} gpu={}", + linear_depth_mismatch, + linear_depth_mismatch == UINT32_MAX + ? 0 + : linear_depth_after[linear_depth_mismatch], + linear_depth_mismatch == UINT32_MAX + ? 0 + : (linear_depth_mismatch % linear_depth_pitch < + linear_depth_width + ? linear_depth_clear + : linear_depth_guest[linear_depth_mismatch]), + linear_stencil_after == linear_stencil_guest, + TextureCacheTestAccess::Contains(texture_cache, + linear_depth_image), + texture_cache.GetImage(linear_depth_image).IsGpuModified()) + .c_str()); - constexpr uint64_t tiled_depth_offset = 0x380000; - constexpr uint32_t tiled_depth_width = 3; - constexpr uint32_t tiled_depth_height = 2; - constexpr uint32_t tiled_depth_pitch = 4; - constexpr uint32_t tiled_depth_layers = 2; - constexpr uint64_t tiled_depth_slice = 0x10000; - constexpr uint64_t tiled_depth_size = tiled_depth_slice * tiled_depth_layers; - constexpr uint32_t tiled_depth_clear = 0x3f200000u; - constexpr uint32_t tiled_depth_stale = 0xdeadbeefu; - std::vector tiled_depth_guest(tiled_depth_size / sizeof(uint32_t), - tiled_depth_stale); - std::memcpy(memory + tiled_depth_offset, tiled_depth_guest.data(), tiled_depth_size); - auto tiled_depth_desc = MakeLinearDesc( - base + tiled_depth_offset, tiled_depth_size, vk::Format::eD32Sfloat, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {tiled_depth_width, tiled_depth_height, 1}, - tiled_depth_layers, 4, 1); - tiled_depth_desc.type = BindingType::DepthTarget; - tiled_depth_desc.info.pitch = tiled_depth_pitch; - tiled_depth_desc.info.tile_mode = Prospero::TileMode::kDepth; - tiled_depth_desc.info.mip_layout[0] = {0, tiled_depth_size, tiled_depth_pitch, - tiled_depth_height}; - tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - tiled_depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc); - Require(name, "tiled depth native clear", - texture_cache.ClearImageFromBuffer(command, tiled_depth_desc.info.data.address, - tiled_depth_desc.info.data.size, - tiled_depth_clear), - "failed to create GPU-current layered tiled depth contents"); - const auto [tiled_prefix, tiled_prefix_offset] = - TextureCacheTestAccess::MapDownload(texture_cache, 64, 64); - Require(name, "tiled depth download prefix", - tiled_prefix != nullptr && tiled_prefix_offset != 0, - "failed to force a nonzero reusable-download offset"); - const auto tiled_depth_tick = scheduler.CurrentTick(); - Require(name, "tiled depth download queue", - TextureCacheTestAccess::TryDownload(texture_cache, tiled_depth_image), - "layered tiled depth readback was rejected"); - std::vector tiled_depth_before(tiled_depth_guest.size()); - std::memcpy(tiled_depth_before.data(), memory + tiled_depth_offset, tiled_depth_size); - Require(name, "tiled depth deferred publication", - tiled_depth_before == tiled_depth_guest && - scheduler.CurrentTick() == tiled_depth_tick, - "tiled depth readback published before GPU completion"); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - std::vector tiled_depth_after(tiled_depth_guest.size()); - std::memcpy(tiled_depth_after.data(), memory + tiled_depth_offset, tiled_depth_size); - const auto active_tiled_words = - tiled_depth_width * tiled_depth_height * tiled_depth_layers; - const auto clear_tiled_words = std::ranges::count(tiled_depth_after, tiled_depth_clear); - const auto stale_tiled_words = std::ranges::count(tiled_depth_after, tiled_depth_stale); - Require(name, "tiled depth backing preservation", - clear_tiled_words == active_tiled_words && - stale_tiled_words == tiled_depth_after.size() - active_tiled_words, - fmt::format("tiled depth readback changed clear={}/{} stale={}/{}", - clear_tiled_words, active_tiled_words, stale_tiled_words, - tiled_depth_after.size() - active_tiled_words) - .c_str()); + constexpr uint64_t tiled_depth_offset = 0x380000; + constexpr uint32_t tiled_depth_width = 3; + constexpr uint32_t tiled_depth_height = 2; + constexpr uint32_t tiled_depth_pitch = 4; + constexpr uint32_t tiled_depth_layers = 2; + constexpr uint64_t tiled_depth_slice = 0x10000; + constexpr uint64_t tiled_depth_size = + tiled_depth_slice * tiled_depth_layers; + constexpr uint32_t tiled_depth_clear = 0x3f200000u; + constexpr uint32_t tiled_depth_stale = 0xdeadbeefu; + std::vector tiled_depth_guest( + tiled_depth_size / sizeof(uint32_t), tiled_depth_stale); + std::memcpy(memory + tiled_depth_offset, tiled_depth_guest.data(), + tiled_depth_size); + auto tiled_depth_desc = MakeLinearDesc( + base + tiled_depth_offset, tiled_depth_size, vk::Format::eD32Sfloat, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 4, 1); + tiled_depth_desc.type = BindingType::DepthTarget; + tiled_depth_desc.info.pitch = tiled_depth_pitch; + tiled_depth_desc.info.tile_mode = Prospero::TileMode::kDepth; + tiled_depth_desc.info.mip_layout[0] = { + 0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height}; + tiled_depth_desc.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + tiled_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto tiled_depth_image = texture_cache.FindImage(tiled_depth_desc); + Require(name, "tiled depth native clear", + texture_cache.ClearImageFromBuffer( + command, tiled_depth_desc.info.data.address, + tiled_depth_desc.info.data.size, tiled_depth_clear), + "failed to create GPU-current layered tiled depth contents"); + const auto [tiled_prefix, tiled_prefix_offset] = + TextureCacheTestAccess::MapDownload(texture_cache, 64, 64); + Require(name, "tiled depth download prefix", + tiled_prefix != nullptr && tiled_prefix_offset != 0, + "failed to force a nonzero reusable-download offset"); + const auto tiled_depth_tick = scheduler.CurrentTick(); + Require( + name, "tiled depth download queue", + TextureCacheTestAccess::TryDownload(texture_cache, tiled_depth_image), + "layered tiled depth readback was rejected"); + std::vector tiled_depth_before(tiled_depth_guest.size()); + std::memcpy(tiled_depth_before.data(), memory + tiled_depth_offset, + tiled_depth_size); + Require(name, "tiled depth deferred publication", + tiled_depth_before == tiled_depth_guest && + scheduler.CurrentTick() == tiled_depth_tick, + "tiled depth readback published before GPU completion"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::vector tiled_depth_after(tiled_depth_guest.size()); + std::memcpy(tiled_depth_after.data(), memory + tiled_depth_offset, + tiled_depth_size); + const auto active_tiled_words = + tiled_depth_width * tiled_depth_height * tiled_depth_layers; + const auto clear_tiled_words = + std::ranges::count(tiled_depth_after, tiled_depth_clear); + const auto stale_tiled_words = + std::ranges::count(tiled_depth_after, tiled_depth_stale); + Require( + name, "tiled depth backing preservation", + clear_tiled_words == active_tiled_words && + stale_tiled_words == + tiled_depth_after.size() - active_tiled_words, + fmt::format("tiled depth readback changed clear={}/{} stale={}/{}", + clear_tiled_words, active_tiled_words, stale_tiled_words, + tiled_depth_after.size() - active_tiled_words) + .c_str()); - TileBlockLayout depth_block {}; - Require(name, "tiled depth block layout", - TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t), depth_block), - "failed to describe the tiled depth test surface"); - std::array tiled_depth_tiles {}; - for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { - const uint64_t offset = tiled_depth_slice * layer; - tiled_depth_tiles[layer] = {depth_block.family, - depth_block.bytes_per_element, - offset, - tiled_depth_slice, - offset, - tiled_depth_slice, - 0, - tiled_depth_width, - tiled_depth_height, - 1, - tiled_depth_pitch}; - tiled_depth_tiles[layer].surface_z = layer; - } - auto tiled_depth_input = - CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after); - auto tiled_depth_linear = TextureCacheTestAccess::Tiler(texture_cache) - .Detile(tiled_depth_input.buffer, 0, tiled_depth_size, - tiled_depth_size, tiled_depth_tiles); - auto tiled_depth_output = - CreateHostBuffer(name, tiled_depth_size, AllFlags, - std::vector(tiled_depth_guest.size(), 0)); - const vk::BufferCopy tiled_depth_copy {tiled_depth_linear.offset, 0, tiled_depth_size}; - scheduler.Current().Handle().copyBuffer( - tiled_depth_linear.buffer, tiled_depth_output.buffer, 1, &tiled_depth_copy); - HostReadBarrier(tiled_depth_output.buffer, tiled_depth_size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - scheduler.FinishCurrent(); - const auto tiled_depth_words = - ReadBuffer(name, tiled_depth_output, tiled_depth_guest.size()); - bool tiled_depth_matches = true; - for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { - for (uint32_t y = 0; y < tiled_depth_height; y++) { - for (uint32_t x = 0; x < tiled_depth_width; x++) { - const auto index = layer * (tiled_depth_slice / sizeof(uint32_t)) + - y * tiled_depth_pitch + x; - tiled_depth_matches &= tiled_depth_words[index] == tiled_depth_clear; - } - } - } - Require(name, "tiled depth layered contents", - tiled_depth_matches && - TextureCacheTestAccess::Contains(texture_cache, tiled_depth_image) && - texture_cache.GetImage(tiled_depth_image).IsGpuModified(), - "tiled depth readback lost a padded slice or retired its image"); - DestroyBuffer(&tiled_depth_output); - DestroyBuffer(&tiled_depth_input); + TileBlockLayout depth_block{}; + Require(name, "tiled depth block layout", + TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint32_t), + depth_block), + "failed to describe the tiled depth test surface"); + std::array tiled_depth_tiles{}; + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + const uint64_t offset = tiled_depth_slice * layer; + tiled_depth_tiles[layer] = {depth_block.family, + depth_block.bytes_per_element, + offset, + tiled_depth_slice, + offset, + tiled_depth_slice, + 0, + tiled_depth_width, + tiled_depth_height, + 1, + tiled_depth_pitch}; + tiled_depth_tiles[layer].surface_z = layer; + } + auto tiled_depth_input = + CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_depth_after); + auto tiled_depth_linear = + TextureCacheTestAccess::Tiler(texture_cache) + .Detile(tiled_depth_input.buffer, 0, tiled_depth_size, + tiled_depth_size, tiled_depth_tiles); + auto tiled_depth_output = + CreateHostBuffer(name, tiled_depth_size, AllFlags, + std::vector(tiled_depth_guest.size(), 0)); + const vk::BufferCopy tiled_depth_copy{tiled_depth_linear.offset, 0, + tiled_depth_size}; + scheduler.Current().Handle().copyBuffer(tiled_depth_linear.buffer, + tiled_depth_output.buffer, 1, + &tiled_depth_copy); + HostReadBarrier(tiled_depth_output.buffer, tiled_depth_size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + const auto tiled_depth_words = + ReadBuffer(name, tiled_depth_output, tiled_depth_guest.size()); + bool tiled_depth_matches = true; + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + for (uint32_t y = 0; y < tiled_depth_height; y++) { + for (uint32_t x = 0; x < tiled_depth_width; x++) { + const auto index = layer * (tiled_depth_slice / sizeof(uint32_t)) + + y * tiled_depth_pitch + x; + tiled_depth_matches &= + tiled_depth_words[index] == tiled_depth_clear; + } + } + } + Require(name, "tiled depth layered contents", + tiled_depth_matches && + TextureCacheTestAccess::Contains(texture_cache, + tiled_depth_image) && + texture_cache.GetImage(tiled_depth_image).IsGpuModified(), + "tiled depth readback lost a padded slice or retired its image"); + DestroyBuffer(&tiled_depth_output); + DestroyBuffer(&tiled_depth_input); - constexpr uint64_t tiled_d16_offset = 0x3a0000; - constexpr uint64_t tiled_d16_stencil_offset = 0x3c0000; - constexpr uint16_t tiled_d16_clear = 0x8000u; - constexpr uint16_t tiled_d16_stale = 0xbeefu; - std::vector tiled_d16_guest(tiled_depth_size / sizeof(uint16_t), - tiled_d16_stale); - constexpr std::array tiled_d16_stencil {0x10, 0x21, 0x32, 0x43, 0x54, 0x65, - 0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, - 0xdc, 0xed, 0xfe, 0x0f}; - std::memcpy(memory + tiled_d16_offset, tiled_d16_guest.data(), tiled_depth_size); - std::memcpy(memory + tiled_d16_stencil_offset, tiled_d16_stencil.data(), - tiled_d16_stencil.size()); - auto tiled_d16_desc = MakeLinearDesc( - base + tiled_d16_offset, tiled_depth_size, vk::Format::eD32SfloatS8Uint, - Prospero::BufferFormat::k16UNorm, - Prospero::ImageType::kColor2D, {tiled_depth_width, tiled_depth_height, 1}, - tiled_depth_layers, 2, 1); - tiled_d16_desc.type = BindingType::DepthTarget; - tiled_d16_desc.info.pitch = tiled_depth_pitch; - tiled_d16_desc.info.tile_mode = Prospero::TileMode::kDepth; - tiled_d16_desc.info.stencil = {base + tiled_d16_stencil_offset, - tiled_d16_stencil.size()}; - tiled_d16_desc.info.mip_layout[0] = {0, tiled_depth_size, tiled_depth_pitch, - tiled_depth_height}; - tiled_d16_desc.view_info.aspect = - vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; - tiled_d16_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc); - Require(name, "tiled D16 native clear", - texture_cache.ClearImageFromBuffer(command, tiled_d16_desc.info.data.address, - tiled_d16_desc.info.data.size, - EncodeD16AsD32(tiled_d16_clear)), - "failed to clear a layered tiled D16 fallback image"); - Require(name, "tiled D16 download queue", - TextureCacheTestAccess::TryDownload(texture_cache, tiled_d16_image), - "layered tiled D16 fallback readback was rejected"); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - std::vector tiled_d16_after(tiled_d16_guest.size()); - std::array tiled_d16_stencil_after {}; - std::memcpy(tiled_d16_after.data(), memory + tiled_d16_offset, tiled_depth_size); - std::memcpy(tiled_d16_stencil_after.data(), memory + tiled_d16_stencil_offset, - tiled_d16_stencil_after.size()); - Require(name, "tiled D16 backing preservation", - std::ranges::count(tiled_d16_after, tiled_d16_clear) == active_tiled_words && - std::ranges::count(tiled_d16_after, tiled_d16_stale) == - tiled_d16_after.size() - active_tiled_words && - tiled_d16_stencil_after == tiled_d16_stencil, - "tiled D16 fallback overwrote inactive depth or stencil backing"); + constexpr uint64_t tiled_d16_offset = 0x3a0000; + constexpr uint64_t tiled_d16_stencil_offset = 0x3c0000; + constexpr uint16_t tiled_d16_clear = 0x8000u; + constexpr uint16_t tiled_d16_stale = 0xbeefu; + std::vector tiled_d16_guest(tiled_depth_size / sizeof(uint16_t), + tiled_d16_stale); + constexpr std::array tiled_d16_stencil{ + 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87, + 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f}; + std::memcpy(memory + tiled_d16_offset, tiled_d16_guest.data(), + tiled_depth_size); + std::memcpy(memory + tiled_d16_stencil_offset, tiled_d16_stencil.data(), + tiled_d16_stencil.size()); + auto tiled_d16_desc = MakeLinearDesc( + base + tiled_d16_offset, tiled_depth_size, + vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k16UNorm, + Prospero::ImageType::kColor2D, + {tiled_depth_width, tiled_depth_height, 1}, tiled_depth_layers, 2, 1); + tiled_d16_desc.type = BindingType::DepthTarget; + tiled_d16_desc.info.pitch = tiled_depth_pitch; + tiled_d16_desc.info.tile_mode = Prospero::TileMode::kDepth; + tiled_d16_desc.info.stencil = {base + tiled_d16_stencil_offset, + tiled_d16_stencil.size()}; + tiled_d16_desc.info.mip_layout[0] = { + 0, tiled_depth_size, tiled_depth_pitch, tiled_depth_height}; + tiled_d16_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + tiled_d16_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto tiled_d16_image = texture_cache.FindImage(tiled_d16_desc); + Require(name, "tiled D16 native clear", + texture_cache.ClearImageFromBuffer( + command, tiled_d16_desc.info.data.address, + tiled_d16_desc.info.data.size, + EncodeD16AsD32(tiled_d16_clear)), + "failed to clear a layered tiled D16 fallback image"); + Require( + name, "tiled D16 download queue", + TextureCacheTestAccess::TryDownload(texture_cache, tiled_d16_image), + "layered tiled D16 fallback readback was rejected"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + std::vector tiled_d16_after(tiled_d16_guest.size()); + std::array tiled_d16_stencil_after{}; + std::memcpy(tiled_d16_after.data(), memory + tiled_d16_offset, + tiled_depth_size); + std::memcpy(tiled_d16_stencil_after.data(), + memory + tiled_d16_stencil_offset, + tiled_d16_stencil_after.size()); + Require(name, "tiled D16 backing preservation", + std::ranges::count(tiled_d16_after, tiled_d16_clear) == + active_tiled_words && + std::ranges::count(tiled_d16_after, tiled_d16_stale) == + tiled_d16_after.size() - active_tiled_words && + tiled_d16_stencil_after == tiled_d16_stencil, + "tiled D16 fallback overwrote inactive depth or stencil backing"); - std::vector tiled_d16_words(tiled_depth_size / sizeof(uint32_t)); - std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(), tiled_depth_size); - std::array tiled_d16_tiles {}; - Require(name, "tiled D16 block layout", - TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t), depth_block), - "failed to describe the tiled D16 test surface"); - for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { - const uint64_t offset = tiled_depth_slice * layer; - tiled_d16_tiles[layer] = {depth_block.family, - depth_block.bytes_per_element, - offset, - tiled_depth_slice, - offset, - tiled_depth_slice, - 0, - tiled_depth_width, - tiled_depth_height, - 1, - tiled_depth_pitch}; - tiled_d16_tiles[layer].surface_z = layer; - } - auto tiled_d16_input = - CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words); - auto tiled_d16_linear = TextureCacheTestAccess::Tiler(texture_cache) - .Detile(tiled_d16_input.buffer, 0, tiled_depth_size, - tiled_depth_size, tiled_d16_tiles); - auto tiled_d16_output = CreateHostBuffer( - name, tiled_depth_size, AllFlags, std::vector(tiled_d16_words.size(), 0)); - const vk::BufferCopy tiled_d16_copy {tiled_d16_linear.offset, 0, tiled_depth_size}; - scheduler.Current().Handle().copyBuffer(tiled_d16_linear.buffer, - tiled_d16_output.buffer, 1, &tiled_d16_copy); - HostReadBarrier(tiled_d16_output.buffer, tiled_depth_size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - scheduler.FinishCurrent(); - const auto tiled_d16_linear_words = - ReadBuffer(name, tiled_d16_output, tiled_d16_words.size()); - std::vector tiled_d16_linear_values(tiled_d16_guest.size()); - std::memcpy(tiled_d16_linear_values.data(), tiled_d16_linear_words.data(), - tiled_depth_size); - bool tiled_d16_matches = true; - for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { - for (uint32_t y = 0; y < tiled_depth_height; y++) { - for (uint32_t x = 0; x < tiled_depth_width; x++) { - const auto index = layer * (tiled_depth_slice / sizeof(uint16_t)) + - y * tiled_depth_pitch + x; - tiled_d16_matches &= tiled_d16_linear_values[index] == tiled_d16_clear; - } - } - } - Require(name, "tiled D16 layered contents", - tiled_d16_matches && - TextureCacheTestAccess::Contains(texture_cache, tiled_d16_image) && - texture_cache.GetImage(tiled_d16_image).IsGpuModified(), - "tiled D16 fallback lost a converted layer or retired its image"); - DestroyBuffer(&tiled_d16_output); - DestroyBuffer(&tiled_d16_input); + std::vector tiled_d16_words(tiled_depth_size / + sizeof(uint32_t)); + std::memcpy(tiled_d16_words.data(), tiled_d16_after.data(), + tiled_depth_size); + std::array tiled_d16_tiles{}; + Require(name, "tiled D16 block layout", + TileGetBlockLayout(TileBlockFamily::Depth64KB, sizeof(uint16_t), + depth_block), + "failed to describe the tiled D16 test surface"); + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + const uint64_t offset = tiled_depth_slice * layer; + tiled_d16_tiles[layer] = {depth_block.family, + depth_block.bytes_per_element, + offset, + tiled_depth_slice, + offset, + tiled_depth_slice, + 0, + tiled_depth_width, + tiled_depth_height, + 1, + tiled_depth_pitch}; + tiled_d16_tiles[layer].surface_z = layer; + } + auto tiled_d16_input = + CreateHostBuffer(name, tiled_depth_size, AllFlags, tiled_d16_words); + auto tiled_d16_linear = + TextureCacheTestAccess::Tiler(texture_cache) + .Detile(tiled_d16_input.buffer, 0, tiled_depth_size, + tiled_depth_size, tiled_d16_tiles); + auto tiled_d16_output = + CreateHostBuffer(name, tiled_depth_size, AllFlags, + std::vector(tiled_d16_words.size(), 0)); + const vk::BufferCopy tiled_d16_copy{tiled_d16_linear.offset, 0, + tiled_depth_size}; + scheduler.Current().Handle().copyBuffer( + tiled_d16_linear.buffer, tiled_d16_output.buffer, 1, &tiled_d16_copy); + HostReadBarrier(tiled_d16_output.buffer, tiled_depth_size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + const auto tiled_d16_linear_words = + ReadBuffer(name, tiled_d16_output, tiled_d16_words.size()); + std::vector tiled_d16_linear_values(tiled_d16_guest.size()); + std::memcpy(tiled_d16_linear_values.data(), tiled_d16_linear_words.data(), + tiled_depth_size); + bool tiled_d16_matches = true; + for (uint32_t layer = 0; layer < tiled_depth_layers; layer++) { + for (uint32_t y = 0; y < tiled_depth_height; y++) { + for (uint32_t x = 0; x < tiled_depth_width; x++) { + const auto index = layer * (tiled_depth_slice / sizeof(uint16_t)) + + y * tiled_depth_pitch + x; + tiled_d16_matches &= + tiled_d16_linear_values[index] == tiled_d16_clear; + } + } + } + Require(name, "tiled D16 layered contents", + tiled_d16_matches && + TextureCacheTestAccess::Contains(texture_cache, + tiled_d16_image) && + texture_cache.GetImage(tiled_d16_image).IsGpuModified(), + "tiled D16 fallback lost a converted layer or retired its image"); + DestroyBuffer(&tiled_d16_output); + DestroyBuffer(&tiled_d16_input); - constexpr uint64_t d16_fallback_offset = 0x33c000; - constexpr uint64_t d16_fallback_stencil_offset = 0x33d000; - constexpr std::array d16_fallback_values {0x0000u, 0x2468u, 0xabcdu, - 0xffffu}; - std::memcpy(memory + d16_fallback_offset, d16_fallback_values.data(), - sizeof(d16_fallback_values)); - std::memset(memory + d16_fallback_stencil_offset, 0x6d, 4); - auto d16_depth_desc = - MakeLinearDesc(base + d16_fallback_offset, sizeof(d16_fallback_values), - vk::Format::eD24UnormS8Uint, - Prospero::BufferFormat::k16UNorm, - Prospero::ImageType::kColor2D, {4, 1, 1}, 1, 2, 1); - d16_depth_desc.type = BindingType::DepthTarget; - d16_depth_desc.info.stencil = {base + d16_fallback_stencil_offset, 4}; - d16_depth_desc.view_info.aspect = - vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; - d16_depth_desc.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto d16_depth_image = texture_cache.FindImage(d16_depth_desc); - (void)texture_cache.FindDepthTarget(d16_depth_image, d16_depth_desc); - texture_cache.MarkGpuWritten(d16_depth_image); - auto d16_storage_desc = d16_depth_desc; - d16_storage_desc.type = BindingType::Storage; - d16_storage_desc.info.stencil = {}; - d16_storage_desc.info.pixel_format = vk::Format::eR16Unorm; - d16_storage_desc.view_info.format = vk::Format::eR16Unorm; - d16_storage_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; - d16_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; - const auto d16_storage_image = texture_cache.FindImage(d16_storage_desc, true); - auto d16_storage_readback = - CreateHostBuffer(name, sizeof(d16_fallback_values), - vk::BufferUsageFlagBits::eTransferDst, std::vector(2, 0)); - auto& d16_storage_native = texture_cache.GetImage(d16_storage_image); - d16_storage_native.Transit(vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits2::eTransferRead, {}, - scheduler.Current().Handle()); - vk::BufferImageCopy d16_storage_copy {}; - d16_storage_copy.bufferRowLength = 4; - d16_storage_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1}; - d16_storage_copy.imageExtent = {4, 1, 1}; - scheduler.Current().Handle().copyImageToBuffer( - d16_storage_native.backing.image, vk::ImageLayout::eTransferSrcOptimal, - d16_storage_readback.buffer, 1, &d16_storage_copy); - HostReadBarrier(d16_storage_readback.buffer, d16_storage_readback.size, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - scheduler.FinishCurrent(); - const auto d16_storage_words = ReadBuffer(name, d16_storage_readback, 2); - std::array d16_storage_values {}; - std::memcpy(d16_storage_values.data(), d16_storage_words.data(), - sizeof(d16_storage_values)); - Require(name, "D16 fallback depth/storage copy", - d16_storage_values == d16_fallback_values && - d16_storage_image != d16_depth_image && - texture_cache.GetImage(d16_storage_image).IsGpuModified(), - "D16 fallback reinterpreted four-byte host depth as two-byte color"); - DestroyBuffer(&d16_storage_readback); + constexpr uint64_t d16_fallback_offset = 0x33c000; + constexpr uint64_t d16_fallback_stencil_offset = 0x33d000; + constexpr std::array d16_fallback_values{0x0000u, 0x2468u, + 0xabcdu, 0xffffu}; + std::memcpy(memory + d16_fallback_offset, d16_fallback_values.data(), + sizeof(d16_fallback_values)); + std::memset(memory + d16_fallback_stencil_offset, 0x6d, 4); + auto d16_depth_desc = MakeLinearDesc( + base + d16_fallback_offset, sizeof(d16_fallback_values), + vk::Format::eD24UnormS8Uint, Prospero::BufferFormat::k16UNorm, + Prospero::ImageType::kColor2D, {4, 1, 1}, 1, 2, 1); + d16_depth_desc.type = BindingType::DepthTarget; + d16_depth_desc.info.stencil = {base + d16_fallback_stencil_offset, 4}; + d16_depth_desc.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + d16_depth_desc.view_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto d16_depth_image = texture_cache.FindImage(d16_depth_desc); + (void)texture_cache.FindDepthTarget(d16_depth_image, d16_depth_desc); + texture_cache.MarkGpuWritten(d16_depth_image); + auto d16_storage_desc = d16_depth_desc; + d16_storage_desc.type = BindingType::Storage; + d16_storage_desc.info.stencil = {}; + d16_storage_desc.info.pixel_format = vk::Format::eR16Unorm; + d16_storage_desc.view_info.format = vk::Format::eR16Unorm; + d16_storage_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + d16_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + const auto d16_storage_image = + texture_cache.FindImage(d16_storage_desc, true); + auto d16_storage_readback = CreateHostBuffer( + name, sizeof(d16_fallback_values), + vk::BufferUsageFlagBits::eTransferDst, std::vector(2, 0)); + auto &d16_storage_native = texture_cache.GetImage(d16_storage_image); + d16_storage_native.Transit(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, {}, + scheduler.Current().Handle()); + vk::BufferImageCopy d16_storage_copy{}; + d16_storage_copy.bufferRowLength = 4; + d16_storage_copy.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, + 0, 1}; + d16_storage_copy.imageExtent = {4, 1, 1}; + scheduler.Current().Handle().copyImageToBuffer( + d16_storage_native.backing.image, + vk::ImageLayout::eTransferSrcOptimal, d16_storage_readback.buffer, 1, + &d16_storage_copy); + HostReadBarrier(d16_storage_readback.buffer, d16_storage_readback.size, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.FinishCurrent(); + const auto d16_storage_words = ReadBuffer(name, d16_storage_readback, 2); + std::array d16_storage_values{}; + std::memcpy(d16_storage_values.data(), d16_storage_words.data(), + sizeof(d16_storage_values)); + Require( + name, "D16 fallback depth/storage copy", + d16_storage_values == d16_fallback_values && + d16_storage_image != d16_depth_image && + texture_cache.GetImage(d16_storage_image).IsGpuModified(), + "D16 fallback reinterpreted four-byte host depth as two-byte color"); + DestroyBuffer(&d16_storage_readback); - if (!texture_cache.GetImage(combined_destination_image).IsGpuModified()) { - texture_cache.MarkGpuWritten(combined_destination_image); - } - constexpr float stale_added_stencil_depth = 0.125f; - Libs::LibKernel::Memory::WriteBacking(combined_destination.info.data.address, - &stale_added_stencil_depth, - sizeof(stale_added_stencil_depth)); - const auto depth_gc_tick = scheduler.CurrentTick(); - TextureCacheTestAccess::ConfigureGarbageCollection( - texture_cache, std::array {combined_destination_image}, 81, 0); - texture_cache.RunGarbageCollector(); - float depth_before_completion = 0.0f; - Libs::LibKernel::Memory::TryReadBacking(combined_destination.info.data.address, - &depth_before_completion, - sizeof(depth_before_completion)); - const bool depth_image_retired = - !TextureCacheTestAccess::Contains(texture_cache, combined_destination_image); - const bool depth_proxy_retired = !texture_cache.FindImageFromRange( - base + added_stencil_offset, added_stencil_size, false); - Require(name, "depth/stencil deferred pressure retirement", - depth_image_retired && depth_proxy_retired && - scheduler.CurrentTick() == depth_gc_tick && - depth_before_completion == stale_added_stencil_depth, - fmt::format("GC failed to retire/defer depth: image={} proxy={} tick={}/{} " - "depth={}/{}", - depth_image_retired, depth_proxy_retired, scheduler.CurrentTick(), - depth_gc_tick, depth_before_completion, stale_added_stencil_depth) - .c_str()); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - float depth_after_completion = 0.0f; - Libs::LibKernel::Memory::TryReadBacking(combined_destination.info.data.address, - &depth_after_completion, - sizeof(depth_after_completion)); - Require(name, "depth/stencil depth-plane preservation", - depth_after_completion == added_stencil_depth_value, - "GC discarded the current depth plane of a depth/stencil " - "image"); + if (!texture_cache.GetImage(combined_destination_image).IsGpuModified()) { + texture_cache.MarkGpuWritten(combined_destination_image); + } + constexpr float stale_added_stencil_depth = 0.125f; + Libs::LibKernel::Memory::WriteBacking( + combined_destination.info.data.address, &stale_added_stencil_depth, + sizeof(stale_added_stencil_depth)); + const auto depth_gc_tick = scheduler.CurrentTick(); + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, std::array{combined_destination_image}, 81, 0); + texture_cache.RunGarbageCollector(); + float depth_before_completion = 0.0f; + Libs::LibKernel::Memory::TryReadBacking( + combined_destination.info.data.address, &depth_before_completion, + sizeof(depth_before_completion)); + const bool depth_image_retired = !TextureCacheTestAccess::Contains( + texture_cache, combined_destination_image); + const bool depth_proxy_retired = !texture_cache.FindImageFromRange( + base + added_stencil_offset, added_stencil_size, false); + Require( + name, "depth/stencil deferred pressure retirement", + depth_image_retired && depth_proxy_retired && + scheduler.CurrentTick() == depth_gc_tick && + depth_before_completion == stale_added_stencil_depth, + fmt::format( + "GC failed to retire/defer depth: image={} proxy={} tick={}/{} " + "depth={}/{}", + depth_image_retired, depth_proxy_retired, scheduler.CurrentTick(), + depth_gc_tick, depth_before_completion, stale_added_stencil_depth) + .c_str()); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + float depth_after_completion = 0.0f; + Libs::LibKernel::Memory::TryReadBacking( + combined_destination.info.data.address, &depth_after_completion, + sizeof(depth_after_completion)); + Require(name, "depth/stencil depth-plane preservation", + depth_after_completion == added_stencil_depth_value, + "GC discarded the current depth plane of a depth/stencil " + "image"); - constexpr size_t gc_depth_pair_count = 6; - constexpr uint64_t gc_depth_pair_offset = 0x350000; - constexpr uint64_t gc_depth_pair_stride = 0x1000; - std::array gc_depth_images {}; - std::array gc_stencil_images {}; - std::array gc_depth_lru {}; - for (size_t index = 0; index < gc_depth_pair_count; index++) { - auto depth = - MakeLinearDesc(base + gc_depth_pair_offset + index * gc_depth_pair_stride, - sizeof(float), vk::Format::eD32SfloatS8Uint, - Prospero::BufferFormat::k32Float, - Prospero::ImageType::kColor2D, {1, 1, 1}, 1, 4, 1); - depth.type = BindingType::DepthTarget; - depth.info.stencil = {depth.info.data.address + gc_depth_pair_stride / 2, - sizeof(uint32_t)}; - depth.view_info.format = vk::Format::eD32SfloatS8Uint; - depth.view_info.aspect = - vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; - depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - gc_depth_images[index] = texture_cache.FindImage(depth); - TextureCacheTestAccess::AssociateStencil(texture_cache, gc_depth_images[index], - depth.info.stencil); - gc_stencil_images[index] = texture_cache.FindImageFromRange( - depth.info.stencil.address, depth.info.stencil.size, false); - gc_depth_lru[index * 2] = gc_depth_images[index]; - gc_depth_lru[index * 2 + 1] = gc_stencil_images[index]; - } - Require( - name, "depth/stencil GC fixtures", - std::ranges::all_of(gc_depth_images, [](ImageId image) { return bool {image}; }) && - std::ranges::all_of(gc_stencil_images, - [](ImageId image) { return bool {image}; }), - "failed to create six depth/stencil association pairs"); - TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache, gc_depth_lru, 81, - UINT64_MAX); - texture_cache.RunGarbageCollector(); - bool gc_depth_budget = true; - for (size_t index = 0; index < gc_depth_pair_count; index++) { - const bool expected_live = index == gc_depth_pair_count - 1; - gc_depth_budget &= - TextureCacheTestAccess::Contains(texture_cache, gc_depth_images[index]) == - expected_live && - TextureCacheTestAccess::Contains(texture_cache, gc_stencil_images[index]) == - expected_live; - } - Require(name, "depth/stencil GC traversal budget", gc_depth_budget, - "recursive association deletion stopped LRU traversal or " - "exceeded the ten-entry deletion budget"); + constexpr size_t gc_depth_pair_count = 6; + constexpr uint64_t gc_depth_pair_offset = 0x350000; + constexpr uint64_t gc_depth_pair_stride = 0x1000; + std::array gc_depth_images{}; + std::array gc_stencil_images{}; + std::array gc_depth_lru{}; + for (size_t index = 0; index < gc_depth_pair_count; index++) { + auto depth = MakeLinearDesc( + base + gc_depth_pair_offset + index * gc_depth_pair_stride, + sizeof(float), vk::Format::eD32SfloatS8Uint, + Prospero::BufferFormat::k32Float, Prospero::ImageType::kColor2D, + {1, 1, 1}, 1, 4, 1); + depth.type = BindingType::DepthTarget; + depth.info.stencil = {depth.info.data.address + + gc_depth_pair_stride / 2, + sizeof(uint32_t)}; + depth.view_info.format = vk::Format::eD32SfloatS8Uint; + depth.view_info.aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + gc_depth_images[index] = texture_cache.FindImage(depth); + TextureCacheTestAccess::AssociateStencil( + texture_cache, gc_depth_images[index], depth.info.stencil); + gc_stencil_images[index] = texture_cache.FindImageFromRange( + depth.info.stencil.address, depth.info.stencil.size, false); + gc_depth_lru[index * 2] = gc_depth_images[index]; + gc_depth_lru[index * 2 + 1] = gc_stencil_images[index]; + } + Require( + name, "depth/stencil GC fixtures", + std::ranges::all_of(gc_depth_images, + [](ImageId image) { return bool{image}; }) && + std::ranges::all_of(gc_stencil_images, + [](ImageId image) { return bool{image}; }), + "failed to create six depth/stencil association pairs"); + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, gc_depth_lru, 81, UINT64_MAX); + texture_cache.RunGarbageCollector(); + bool gc_depth_budget = true; + for (size_t index = 0; index < gc_depth_pair_count; index++) { + const bool expected_live = index == gc_depth_pair_count - 1; + gc_depth_budget &= + TextureCacheTestAccess::Contains( + texture_cache, gc_depth_images[index]) == expected_live && + TextureCacheTestAccess::Contains( + texture_cache, gc_stencil_images[index]) == expected_live; + } + Require(name, "depth/stencil GC traversal budget", gc_depth_budget, + "recursive association deletion stopped LRU traversal or " + "exceeded the ten-entry deletion budget"); - constexpr uint64_t large_offset = 0x400000; - constexpr uint32_t large_width = 4096; - constexpr uint32_t large_height = 2047; - constexpr uint64_t large_size = - uint64_t {large_width} * large_height * sizeof(uint32_t); - static_assert(large_size < (32ull << 20)); - std::memset(memory + large_offset, 0, static_cast(large_size)); - auto large_desc = MakeLinearDesc( - base + large_offset, large_size, vk::Format::eR8G8B8A8Unorm, - Prospero::BufferFormat::k8_8_8_8UNorm, - Prospero::ImageType::kColor2D, {large_width, large_height, 1}, 1, 4, 1); - const auto RunLargeReadback = [&](uint32_t clear_value) { - const auto image = texture_cache.FindImage(large_desc); - Require(name, "near-capacity image clear", - texture_cache.ClearImageFromBuffer(command, large_desc.info.data.address, - large_desc.info.data.size, clear_value), - "failed to write the near-capacity readback source image"); - const std::array sample_offsets {0, large_size / 2, - large_size - sizeof(uint32_t)}; - constexpr uint32_t stale = 0; - for (const auto offset: sample_offsets) { - Libs::LibKernel::Memory::WriteBacking(base + large_offset + offset, &stale, - sizeof(stale)); - } - TextureCacheTestAccess::ConfigureGarbageCollection(texture_cache, - std::array {image}, 81, 0); - const auto tick = scheduler.CurrentTick(); - texture_cache.RunGarbageCollector(); - const auto handle = - BufferCacheTestAccess::DownloadBuffer(resources.GetBufferCache()).Handle(); - Require(name, "near-capacity deferred retirement", - !TextureCacheTestAccess::Contains(texture_cache, image) && - scheduler.CurrentTick() == tick, - "near-capacity readback was rejected or synchronously submitted"); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - bool content = true; - for (const auto offset: sample_offsets) { - uint32_t value = 0; - Libs::LibKernel::Memory::TryReadBacking(base + large_offset + offset, &value, - sizeof(value)); - content &= value == clear_value; - } - Require(name, "near-capacity readback content", content, - "near-capacity image readback did not publish its GPU contents"); - return handle; - }; - const auto large_download = RunLargeReadback(0xa5a5a5a5u); - const auto reused_large_download = RunLargeReadback(0x5a5a5a5au); - Require(name, "near-capacity shared download reuse", - large_download == reused_large_download, - "successive near-capacity image transfers replaced the shared " - "download buffer"); + constexpr uint64_t large_offset = 0x400000; + constexpr uint32_t large_width = 4096; + constexpr uint32_t large_height = 2047; + constexpr uint64_t large_size = + uint64_t{large_width} * large_height * sizeof(uint32_t); + static_assert(large_size < (32ull << 20)); + std::memset(memory + large_offset, 0, static_cast(large_size)); + auto large_desc = MakeLinearDesc( + base + large_offset, large_size, vk::Format::eR8G8B8A8Unorm, + Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D, + {large_width, large_height, 1}, 1, 4, 1); + const auto RunLargeReadback = [&](uint32_t clear_value) { + const auto image = texture_cache.FindImage(large_desc); + Require(name, "near-capacity image clear", + texture_cache.ClearImageFromBuffer( + command, large_desc.info.data.address, + large_desc.info.data.size, clear_value), + "failed to write the near-capacity readback source image"); + const std::array sample_offsets{ + 0, large_size / 2, large_size - sizeof(uint32_t)}; + constexpr uint32_t stale = 0; + for (const auto offset : sample_offsets) { + Libs::LibKernel::Memory::WriteBacking(base + large_offset + offset, + &stale, sizeof(stale)); + } + TextureCacheTestAccess::ConfigureGarbageCollection( + texture_cache, std::array{image}, 81, 0); + const auto tick = scheduler.CurrentTick(); + texture_cache.RunGarbageCollector(); + const auto handle = + BufferCacheTestAccess::DownloadBuffer(resources.GetBufferCache()) + .Handle(); + Require( + name, "near-capacity deferred retirement", + !TextureCacheTestAccess::Contains(texture_cache, image) && + scheduler.CurrentTick() == tick, + "near-capacity readback was rejected or synchronously submitted"); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + bool content = true; + for (const auto offset : sample_offsets) { + uint32_t value = 0; + Libs::LibKernel::Memory::TryReadBacking(base + large_offset + offset, + &value, sizeof(value)); + content &= value == clear_value; + } + Require( + name, "near-capacity readback content", content, + "near-capacity image readback did not publish its GPU contents"); + return handle; + }; + const auto large_download = RunLargeReadback(0xa5a5a5a5u); + const auto reused_large_download = RunLargeReadback(0x5a5a5a5au); + Require(name, "near-capacity shared download reuse", + large_download == reused_large_download, + "successive near-capacity image transfers replaced the shared " + "download buffer"); - constexpr uint64_t tile_alias_offset = 0x2000000; - constexpr uint64_t tile_alias_size = 0x400000; - constexpr uint32_t tile_alias_extent = 1024; - std::memset(memory + tile_alias_offset, 0, static_cast(tile_alias_size)); - auto render_target_alias = MakeLinearDesc( - base + tile_alias_offset, tile_alias_size, vk::Format::eR8G8B8A8Unorm, - Prospero::BufferFormat::k8_8_8_8UNorm, - Prospero::ImageType::kColor2D, {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1); - render_target_alias.type = BindingType::Storage; - render_target_alias.info.tile_mode = - Prospero::TileMode::kRenderTarget; - render_target_alias.view_info.usage = vk::ImageUsageFlagBits::eStorage; - const auto render_target_alias_image = texture_cache.FindImage(render_target_alias); + constexpr uint64_t tile_alias_offset = 0x2000000; + constexpr uint64_t tile_alias_size = 0x400000; + constexpr uint32_t tile_alias_extent = 1024; + std::memset(memory + tile_alias_offset, 0, + static_cast(tile_alias_size)); + auto render_target_alias = MakeLinearDesc( + base + tile_alias_offset, tile_alias_size, vk::Format::eR8G8B8A8Unorm, + Prospero::BufferFormat::k8_8_8_8UNorm, Prospero::ImageType::kColor2D, + {tile_alias_extent, tile_alias_extent, 1}, 1, 4, 1); + render_target_alias.type = BindingType::Storage; + render_target_alias.info.tile_mode = Prospero::TileMode::kRenderTarget; + render_target_alias.view_info.usage = vk::ImageUsageFlagBits::eStorage; + const auto render_target_alias_image = + texture_cache.FindImage(render_target_alias); - auto standard_4kb_alias = render_target_alias; - standard_4kb_alias.type = BindingType::Texture; - standard_4kb_alias.info.tile_mode = - Prospero::TileMode::kStandard4KB; - standard_4kb_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto standard_4kb_alias_image = texture_cache.FindImage(standard_4kb_alias); - auto repeated_standard_4kb_alias = standard_4kb_alias; - const auto repeated_standard_4kb_alias_image = - texture_cache.FindImage(repeated_standard_4kb_alias); - Require( - name, "equal-size tile-mode alias", - render_target_alias_image && - standard_4kb_alias_image && - standard_4kb_alias_image != - render_target_alias_image && - repeated_standard_4kb_alias_image == - standard_4kb_alias_image && - texture_cache - .GetImage(render_target_alias_image) - .info.tile_mode == - Prospero::TileMode::kRenderTarget && - texture_cache.GetImage(standard_4kb_alias_image) - .info.tile_mode == - Prospero::TileMode::kStandard4KB, - "equal address/size lookup reused an incompatible " - "tiled backing"); + auto standard_4kb_alias = render_target_alias; + standard_4kb_alias.type = BindingType::Texture; + standard_4kb_alias.info.tile_mode = Prospero::TileMode::kStandard4KB; + standard_4kb_alias.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto standard_4kb_alias_image = + texture_cache.FindImage(standard_4kb_alias); + auto repeated_standard_4kb_alias = standard_4kb_alias; + const auto repeated_standard_4kb_alias_image = + texture_cache.FindImage(repeated_standard_4kb_alias); + Require( + name, "equal-size tile-mode alias", + render_target_alias_image && standard_4kb_alias_image && + standard_4kb_alias_image != render_target_alias_image && + repeated_standard_4kb_alias_image == standard_4kb_alias_image && + texture_cache.GetImage(render_target_alias_image) + .info.tile_mode == Prospero::TileMode::kRenderTarget && + texture_cache.GetImage(standard_4kb_alias_image).info.tile_mode == + Prospero::TileMode::kStandard4KB, + "equal address/size lookup reused an incompatible " + "tiled backing"); - for (auto &output : ms_observer_outputs) { - DestroyBuffer(&output); - } - DestroyBuffer(&layered_readback); - DestroyBuffer(&bgra16_readback); - DestroyBuffer(&mixed_source_readback); - DestroyBuffer(&mip_formatted_readback); - DestroyBuffer(&mip_prefix_readback); - DestroyBuffer(&partial_cpu_refresh_readback); - DestroyBuffer(&mirror_cpu_readback); - DestroyBuffer(&exact_buffer_readback); - m_device.destroyDescriptorPool(observer_pool, nullptr); - m_device.destroyPipeline(observer_depth_pipeline, - nullptr); - m_device.destroyPipelineLayout(observer_pipeline_layout, - nullptr); - m_device.destroyDescriptorSetLayout(observer_set_layout, - nullptr); - m_device.destroyShaderModule(ms_depth_module, nullptr); + for (auto &output : ms_observer_outputs) { + DestroyBuffer(&output); + } + DestroyBuffer(&layered_readback); + DestroyBuffer(&bgra16_readback); + DestroyBuffer(&mixed_source_readback); + DestroyBuffer(&mip_formatted_readback); + DestroyBuffer(&mip_prefix_readback); + DestroyBuffer(&partial_cpu_refresh_readback); + DestroyBuffer(&mirror_cpu_readback); + DestroyBuffer(&exact_buffer_readback); + m_device.destroyDescriptorPool(observer_pool, nullptr); + m_device.destroyPipeline(observer_depth_pipeline, nullptr); + m_device.destroyPipelineLayout(observer_pipeline_layout, nullptr); + m_device.destroyDescriptorSetLayout(observer_set_layout, nullptr); + m_device.destroyShaderModule(ms_depth_module, nullptr); - resources.SetGpu(nullptr); - resources.UnmapMemory(base, allocation_size); - scheduler.Finish(); - } - context.ShutdownGpu(); - Require(name, "unmap direct backing", - Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, - "cache direct-memory mapping release failed"); - Require( - name, "release direct backing", - Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0, - "cache direct-memory allocation release failed"); - std::printf("[host] %-32s ok\n", name); - } + resources.SetGpu(nullptr); + resources.UnmapMemory(base, allocation_size); + scheduler.Finish(); + } + context.ShutdownGpu(); + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "cache direct-memory mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "cache direct-memory allocation release failed"); + std::printf("[host] %-32s ok\n", name); + } - void CheckBgra16Readback() { - constexpr const char* name = "Bgra16Readback"; - constexpr uintptr_t base = 0x0000000204000000ull; - constexpr uint64_t allocation_size = 0x200000; - constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float; - constexpr auto tile = Prospero::TileMode::kRenderTarget; - EnsureRuntimeContext(); + void CheckBgra16Readback() { + constexpr const char *name = "Bgra16Readback"; + constexpr uintptr_t base = 0x0000000204000000ull; + constexpr uint64_t allocation_size = 0x200000; + constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float; + constexpr auto tile = Prospero::TileMode::kRenderTarget; + EnsureRuntimeContext(); - int64_t direct_offset = -1; - Require(name, "direct allocation", - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, - allocation_size, 0, &direct_offset) == 0, - "BGRA16 direct-memory allocation failed"); - void* mapped = reinterpret_cast(base); - Require(name, "direct mapping", - Libs::LibKernel::Memory::KernelMapDirectMemory( - &mapped, allocation_size, 0x3, 0x10, direct_offset, allocation_size) == 0 && - mapped == reinterpret_cast(base), - "BGRA16 fixed mapping failed"); + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_size, 0, &direct_offset) == 0, + "BGRA16 direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_size) == 0 && + mapped == reinterpret_cast(base), + "BGRA16 fixed mapping failed"); - CommandScheduler scheduler(Renderer(), m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - { - GpuResourceManager resources(m_runtime_context, scheduler); - resources.MapMemory(base, allocation_size); - const uint32_t pitch = TileGetTexturePitch(format, 1, tile); - TileSizeAlign total {}; - TileSizeOffset mip {}; - TilePaddedSize padded {}; - TileGetTextureSize(format, 1, 1, 1, tile, &total, &mip, &padded); - Require(name, "tiled layout", total.size >= 8 && total.size <= allocation_size, - "BGRA16 tiled layout exceeds its guest allocation"); - std::memset(mapped, 0x5a, total.size); + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + { + GpuResourceManager resources(m_runtime_context, scheduler); + resources.MapMemory(base, allocation_size); + const uint32_t pitch = TileGetTexturePitch(format, 1, tile); + TileSizeAlign total{}; + TileSizeOffset mip{}; + TilePaddedSize padded{}; + TileGetTextureSize(format, 1, 1, 1, tile, &total, &mip, &padded); + Require(name, "tiled layout", + total.size >= 8 && total.size <= allocation_size, + "BGRA16 tiled layout exceeds its guest allocation"); + std::memset(mapped, 0x5a, total.size); - ImageDesc desc {}; - desc.type = BindingType::RenderTarget; - desc.info.data = {base, total.size}; - desc.info.pixel_format = vk::Format::eR16G16B16A16Sfloat; - desc.info.guest_format = format; - desc.info.type = Prospero::ImageType::kColor2D; - desc.info.extent = {1, 1, 1}; - desc.info.resources = {1, 1}; - desc.info.pitch = pitch; - desc.info.bytes_per_block = 8; - desc.info.samples = 1; - desc.info.tile_mode = tile; - desc.info.bgra16 = true; - desc.info.mip_layout[0] = {mip.offset, mip.size, pitch, 1}; - desc.view_info.format = desc.info.pixel_format; - desc.view_info.type = vk::ImageViewType::e2D; - desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; - desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + ImageDesc desc{}; + desc.type = BindingType::RenderTarget; + desc.info.data = {base, total.size}; + desc.info.pixel_format = vk::Format::eR16G16B16A16Sfloat; + desc.info.guest_format = format; + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = {1, 1, 1}; + desc.info.resources = {1, 1}; + desc.info.pitch = pitch; + desc.info.bytes_per_block = 8; + desc.info.samples = 1; + desc.info.tile_mode = tile; + desc.info.bgra16 = true; + desc.info.mip_layout[0] = {mip.offset, mip.size, pitch, 1}; + desc.view_info.format = desc.info.pixel_format; + desc.view_info.type = vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - auto& cache = resources.GetTextureCache(); - const auto id = cache.FindImage(desc); - auto& image = cache.GetImage(id); - image.Transit(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, - {}, scheduler.Current().Handle()); - vk::ClearColorValue clear {}; - clear.float32[0] = 1.0f; - clear.float32[1] = 2.0f; - clear.float32[2] = 3.0f; - clear.float32[3] = 4.0f; - const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}; - scheduler.Current().Handle().clearColorImage( - image.backing.image, vk::ImageLayout::eTransferDstOptimal, clear, range); - cache.MarkGpuWritten(id); - Require(name, "guest readback queue", TextureCacheTestAccess::TryDownload(cache, id), - "tiled BGRA16 guest readback was rejected"); - auto mirror = resources.GetBufferCache().ObtainBuffer(scheduler.Current(), base, - total.size, false, true, true); - Require(name, "mirror owner", mirror.buffer != nullptr && mirror.owner != nullptr, - "tiled BGRA16 mirror has no BufferCache owner"); - scheduler.Current().RetainResourceUntilFence(mirror.owner); - auto mirror_readback = - CreateHostBuffer(name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0}); - const vk::BufferCopy copy {mirror.offset, 0, 8}; - scheduler.Current().Handle().copyBuffer(mirror.buffer, mirror_readback.buffer, 1, - ©); - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = mirror_readback.buffer; - barrier.size = mirror_readback.size; - scheduler.Current().Handle().pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eHost, {}, 0, - nullptr, 1, &barrier, 0, nullptr); - scheduler.Finish(); - scheduler.DrainPriorityOperations(); + auto &cache = resources.GetTextureCache(); + const auto id = cache.FindImage(desc); + auto &image = cache.GetImage(id); + image.Transit(vk::ImageLayout::eTransferDstOptimal, + vk::AccessFlagBits2::eTransferWrite, {}, + scheduler.Current().Handle()); + vk::ClearColorValue clear{}; + clear.float32[0] = 1.0f; + clear.float32[1] = 2.0f; + clear.float32[2] = 3.0f; + clear.float32[3] = 4.0f; + const vk::ImageSubresourceRange range{vk::ImageAspectFlagBits::eColor, 0, + 1, 0, 1}; + scheduler.Current().Handle().clearColorImage( + image.backing.image, vk::ImageLayout::eTransferDstOptimal, clear, + range); + cache.MarkGpuWritten(id); + Require(name, "guest readback queue", + TextureCacheTestAccess::TryDownload(cache, id), + "tiled BGRA16 guest readback was rejected"); + auto mirror = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), base, total.size, false, true, true); + Require(name, "mirror owner", + mirror.buffer != nullptr && mirror.owner != nullptr, + "tiled BGRA16 mirror has no BufferCache owner"); + scheduler.Current().RetainResourceUntilFence(mirror.owner); + auto mirror_readback = CreateHostBuffer( + name, 8, vk::BufferUsageFlagBits::eTransferDst, {0, 0}); + const vk::BufferCopy copy{mirror.offset, 0, 8}; + scheduler.Current().Handle().copyBuffer(mirror.buffer, + mirror_readback.buffer, 1, ©); + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = mirror_readback.buffer; + barrier.size = mirror_readback.size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, + nullptr); + scheduler.Finish(); + scheduler.DrainPriorityOperations(); - std::vector backing(total.size); - Require(name, "guest backing", - Libs::LibKernel::Memory::TryReadBacking(base, backing.data(), backing.size()), - "tiled BGRA16 guest backing is unreadable"); - const std::array expected {0x4200u, 0x4000u, 0x3c00u, 0x4400u}; - std::array observed {}; - std::memcpy(observed.data(), backing.data(), sizeof(observed)); - Require(name, "guest component order", - observed == expected && - std::all_of(backing.begin() + sizeof(observed), backing.end(), - [](uint8_t value) { return value == 0x5a; }), - "tiled BGRA16 guest readback changed component order or padding"); - Require(name, "mirror component order", - ReadBuffer(name, mirror_readback, 2) == - std::vector {0x40004200u, 0x44003c00u}, - "tiled BGRA16 Buffer mirror changed guest component order"); - DestroyBuffer(&mirror_readback); - resources.UnmapMemory(base, allocation_size); - scheduler.Finish(); - } - Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, - "BGRA16 fixed mapping release failed"); - Require( - name, "release", - Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0, - "BGRA16 direct-memory release failed"); - std::printf("[gpu] %-32s ok\n", name); - } + std::vector backing(total.size); + Require(name, "guest backing", + Libs::LibKernel::Memory::TryReadBacking(base, backing.data(), + backing.size()), + "tiled BGRA16 guest backing is unreadable"); + const std::array expected{0x4200u, 0x4000u, 0x3c00u, + 0x4400u}; + std::array observed{}; + std::memcpy(observed.data(), backing.data(), sizeof(observed)); + Require(name, "guest component order", + observed == expected && + std::all_of(backing.begin() + sizeof(observed), backing.end(), + [](uint8_t value) { return value == 0x5a; }), + "tiled BGRA16 guest readback changed component order or padding"); + Require(name, "mirror component order", + ReadBuffer(name, mirror_readback, 2) == + std::vector{0x40004200u, 0x44003c00u}, + "tiled BGRA16 Buffer mirror changed guest component order"); + DestroyBuffer(&mirror_readback); + resources.UnmapMemory(base, allocation_size); + scheduler.Finish(); + } + Require(name, "unmap", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "BGRA16 fixed mapping release failed"); + Require(name, "release", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "BGRA16 direct-memory release failed"); + std::printf("[gpu] %-32s ok\n", name); + } - void CheckRenderExecutorColorVolumeDiscovery() { - constexpr const char* name = "RenderExecutorColorVolumeDiscovery"; - constexpr uintptr_t base = 0x0000000203e00000ull; - constexpr uint64_t allocation_size = 0x200000; - constexpr uint64_t allocation_alignment = 0x10000; - EnsureRuntimeContext(); + void CheckRenderExecutorColorVolumeDiscovery() { + constexpr const char *name = "RenderExecutorColorVolumeDiscovery"; + constexpr uintptr_t base = 0x0000000203e00000ull; + constexpr uint64_t allocation_size = 0x200000; + constexpr uint64_t allocation_alignment = 0x10000; + EnsureRuntimeContext(); - int64_t direct_offset = -1; - Require(name, "direct allocation", - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, - allocation_alignment, 0, &direct_offset) == 0, - "color-volume direct-memory allocation failed"); - void* mapped = reinterpret_cast(base); - Require(name, "direct mapping", - Libs::LibKernel::Memory::KernelMapDirectMemory(&mapped, allocation_size, 0x3, 0x10, - direct_offset, - allocation_alignment) == 0 && - mapped == reinterpret_cast(base), - "color-volume fixed mapping failed"); - std::memset(mapped, 0, allocation_size); - constexpr uint64_t slice_size = 0x10000; - std::memset(static_cast(mapped) + 31 * slice_size, 0x5a, slice_size); + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "color-volume direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "color-volume fixed mapping failed"); + std::memset(mapped, 0, allocation_size); + constexpr uint64_t slice_size = 0x10000; + std::memset(static_cast(mapped) + 31 * slice_size, 0x5a, + slice_size); - { - RenderContext context(m_runtime_context); - auto& scheduler = context.GetCommandScheduler(); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - registers.SetColorBase(0, {.addr = base}); - registers.SetColorInfo( - 0, {.format = Prospero::ChannelLayout::k10_10_10_2, - .channel_type = Prospero::ChannelType::kUNorm, - .channel_order = Prospero::ChannelOrder::kStandard}); - registers.SetColorAttrib2(0, {.height = 31, .width = 31}); - registers.SetColorAttrib3( - 0, {.depth = 31, - .tile_mode = Prospero::TileMode::kRenderTarget, - .dimension = 2, - .cmask_pipe_aligned = true, - .dcc_pipe_aligned = true}); - registers.SetRenderTargetMask(0x0f); - scheduler.Begin(registers, user_config, shaders); + { + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + registers.SetColorBase(0, {.addr = base}); + registers.SetColorInfo( + 0, {.format = Prospero::ChannelLayout::k10_10_10_2, + .channel_type = Prospero::ChannelType::kUNorm, + .channel_order = Prospero::ChannelOrder::kStandard}); + registers.SetColorAttrib2(0, {.height = 31, .width = 31}); + registers.SetColorAttrib3(0, + {.depth = 31, + .tile_mode = Prospero::TileMode::kRenderTarget, + .dimension = 2, + .cmask_pipe_aligned = true, + .dcc_pipe_aligned = true}); + registers.SetRenderTargetMask(0x0f); + scheduler.Begin(registers, user_config, shaders); - auto& resources = context.GetGpuResources(); - auto& texture_cache = resources.GetTextureCache(); - auto& executor = context.GetRenderExecutor(); - resources.MapMemory(base, allocation_size); + auto &resources = context.GetGpuResources(); + auto &texture_cache = resources.GetTextureCache(); + auto &executor = context.GetRenderExecutor(); + resources.MapMemory(base, allocation_size); - RenderColorInfo color {}; - RenderExecutorTestAccess::ResolveRenderColorTarget(executor, 1, scheduler.Current(), - color, 0); - const auto attachment = texture_cache.FindRenderTarget(color.image_id, color.desc); - const auto& image = texture_cache.GetImage(color.image_id); - Require(name, "captured 3D target", - color.image_id && attachment != nullptr && - color.desc.info.type == Prospero::ImageType::kColor3D && - color.desc.info.extent == vk::Extent3D {32, 32, 32} && - color.desc.info.resources == ImageSubresources {1, 1} && - color.desc.info.pitch == 128 && - color.desc.info.data.size == allocation_size && - color.desc.info.mip_layout[0].size == 0x10000 && - color.desc.view_info.type == vk::ImageViewType::e2D && - color.desc.view_info.layer_count == 1 && - image.backing.image_type == vk::ImageType::e3D && - static_cast(image.backing.flags & - vk::ImageCreateFlagBits::e2DArrayCompatible) && - image.usage.render_target && image.IsGpuModified(), + RenderColorInfo color{}; + RenderExecutorTestAccess::ResolveRenderColorTarget( + executor, 1, scheduler.Current(), color, 0); + const auto attachment = + texture_cache.FindRenderTarget(color.image_id, color.desc); + const auto &image = texture_cache.GetImage(color.image_id); + Require( + name, "captured 3D target", + color.image_id && attachment != nullptr && + color.desc.info.type == Prospero::ImageType::kColor3D && + color.desc.info.extent == vk::Extent3D{32, 32, 32} && + color.desc.info.resources == ImageSubresources{1, 1} && + color.desc.info.pitch == 128 && + color.desc.info.data.size == allocation_size && + color.desc.info.mip_layout[0].size == 0x10000 && + color.desc.view_info.type == vk::ImageViewType::e2D && + color.desc.view_info.layer_count == 1 && + image.backing.image_type == vk::ImageType::e3D && + static_cast(image.backing.flags & + vk::ImageCreateFlagBits::e2DArrayCompatible) && + image.usage.render_target && image.IsGpuModified(), "dimension=2/depth=31 did not create the SDK-defined 32x32x32 " - "backing and 2D " - "attachment slice"); - - RenderExecutorTestAccess::ResetBindings(executor); - registers.SetColorView( - 0, {.base_array_slice_index = 7, .last_array_slice_index = 7}); - RenderColorInfo sliced_color {}; - RenderExecutorTestAccess::ResolveRenderColorTarget( - executor, 2, scheduler.Current(), sliced_color, 0); - RenderDepthInfo no_depth {}; - const auto sliced_rendering = RenderExecutorTestAccess::AcquireRenderTargets( - executor, scheduler.Current(), &sliced_color, 1, no_depth); - Require(name, "3D slice transition", - sliced_color.image_id == color.image_id && sliced_color.image_view != nullptr && - sliced_color.desc.view_info.base_layer == 7 && - sliced_rendering.num_color_attachments == 1 && sliced_rendering.num_layers == 1, - "a nonzero 3D attachment slice was treated as a Vulkan array layer"); - - auto storage_desc = color.desc; - storage_desc.type = BindingType::Storage; - storage_desc.info.guest_format = - Prospero::BufferFormat::k10_10_10_2UNorm; - storage_desc.view_info.type = vk::ImageViewType::e3D; - storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; - const auto storage_id = texture_cache.FindImage(storage_desc); - const auto storage_view = texture_cache.FindTexture(storage_id, storage_desc); - const auto& shared_image = texture_cache.GetImage(storage_id); - Require(name, "storage alias reuse", - storage_id == color.image_id && storage_view != nullptr && - shared_image.IsGpuModified() && shared_image.usage.render_target, - "the matching 3D storage binding did not reuse the live " - "render-target image"); - - Require(name, "volume readback queue", - TextureCacheTestAccess::TryDownload(texture_cache, storage_id), - "the 3D render target could not be queued for guest-layout readback"); - auto mirror = resources.GetBufferCache().ObtainBuffer( - scheduler.Current(), base, allocation_size, false, true, true); - Require(name, "volume mirror", - mirror.buffer != nullptr && mirror.owner != nullptr, - "the 3D render-target readback has no BufferCache owner"); - scheduler.Current().RetainResourceUntilFence(mirror.owner); - auto slice_probe = - CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0}); - const vk::BufferCopy copy {mirror.offset + 31 * slice_size, 0, 4}; - scheduler.Current().Handle().copyBuffer(mirror.buffer, slice_probe.buffer, 1, ©); - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = slice_probe.buffer; - barrier.size = slice_probe.size; - scheduler.Current().Handle().pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, - 1, &barrier, 0, nullptr); - scheduler.Finish(); - scheduler.DrainPriorityOperations(); - Require(name, "volume Z transfer", - ReadBuffer(name, slice_probe, 1) == std::vector {0x5a5a5a5a}, - "render-target upload/readback lost the final Z slice"); - DestroyBuffer(&slice_probe); - - RenderExecutorTestAccess::ResetBindings(executor); - resources.UnmapMemory(base, allocation_size); - scheduler.Finish(); - } - - Require(name, "unmap direct backing", - Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, - "color-volume direct mapping release failed"); - Require( - name, "release direct backing", - Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0, - "color-volume direct-memory allocation release failed"); - std::printf("[gpu] %-32s ok\n", name); - } - - void CheckRenderExecutorStencilBindingDiscovery() { - constexpr const char* name = "RenderExecutorStencilBindingDiscovery"; - constexpr uintptr_t base = 0x0000000203600000ull; - constexpr uint64_t allocation_size = 0x180000; - constexpr uint64_t allocation_alignment = 0x10000; - constexpr uint64_t depth_address = base + 0x40000; - constexpr uint64_t stencil_address = base + 0x70000; - EnsureRuntimeContext(); - - int64_t direct_offset = -1; - Require(name, "direct allocation", - Libs::LibKernel::Memory::KernelAllocateDirectMemory( - 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), allocation_size, - allocation_alignment, 0, &direct_offset) == 0, - "descriptor discovery direct-memory allocation failed"); - void* mapped = reinterpret_cast(base); - Require(name, "direct mapping", - Libs::LibKernel::Memory::KernelMapDirectMemory(&mapped, allocation_size, 0x3, 0x10, - direct_offset, - allocation_alignment) == 0 && - mapped == reinterpret_cast(base), - "descriptor discovery fixed mapping failed"); - std::memset(mapped, 0, allocation_size); - - { - RenderContext context(m_runtime_context); - auto& scheduler = context.GetCommandScheduler(); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - auto& resources = context.GetGpuResources(); - auto& texture_cache = resources.GetTextureCache(); - auto& executor = context.GetRenderExecutor(); - resources.MapMemory(base, allocation_size); - - constexpr auto stencil_format = Prospero::BufferFormat::k8UInt; - constexpr auto linear = Prospero::TileMode::kLinear; - TileSizeAlign stencil_layout {}; - TileGetTextureTotalSize(stencil_format, 1, 1, 1, 1, linear, false, - stencil_layout); - Require(name, "stencil footprint", - stencil_layout.size != 0 && stencil_layout.align != 0 && - (stencil_address & (stencil_layout.align - 1u)) == 0, - "linear stencil descriptor produced an invalid footprint"); - - ShaderRecompiler::IR::Program null_program {}; - null_program.stage = ShaderType::Compute; - null_program.resource_tracking_complete = true; - ShaderRecompiler::IR::ImageResource null_resource {}; - null_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; - null_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - null_resource.read = true; - null_program.info.images.push_back(null_resource); - auto null_volume = null_resource; - null_volume.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; - null_program.info.images.push_back(null_volume); - auto null_storage = null_resource; - null_storage.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - null_storage.read = false; - null_storage.written = true; - null_program.info.images.push_back(null_storage); - ShaderRecompiler::IR::ResourceSnapshot null_snapshot {}; - ShaderRecompiler::IR::DescriptorValue null_descriptor {}; - null_descriptor.dword_count = 8; - null_snapshot.images.assign(3, null_descriptor); - std::string null_error; - Require( - name, "null specialization", - ShaderRecompiler::IR::SpecializeResources(null_program, null_snapshot, &null_error), - null_error.c_str()); - ShaderStageRuntime null_runtime { - std::make_shared(std::move(null_program)), - std::make_shared( - std::move(null_snapshot))}; - auto null_bindings = executor.PrepareBindings(scheduler.Current(), null_runtime, - vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), null_bindings); - Require(name, "null descriptor count", null_bindings.resources.images.size() == 3, - "null descriptor preparation lost an image binding"); - const auto null_image_id = null_bindings.resources.images[0].image_id; - const auto& null_image = texture_cache.GetImage(null_image_id); - Require(name, "one null image per format", - null_bindings.resources.images[1].image_id == null_image_id && - null_bindings.resources.images[2].image_id == null_image_id && - TextureCacheTestAccess::NullImageCount(texture_cache) == 1, - "null view dimension or usage created another image allocation"); - Require(name, "null descriptors share final cache acquisition", - null_image.info.data.Empty() && !null_image.binding.is_bound && - !null_image.binding.is_target && !null_image.binding.needs_rebind && - null_image.usage.texture && null_image.usage.storage && - null_image.IsGpuModified() && - !TextureCacheTestAccess::PendingDownload(texture_cache, null_image_id), - "the shared null image did not preserve texture/storage " - "acquisition without guest readback or RenderExecutor ownership"); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), null_bindings); - Require(name, "null descriptor general layouts", - std::ranges::all_of(null_bindings.resources.images, - [](const auto& binding) { - const auto info = - DescriptorCacheTestAccess::MakeImageInfo(binding); - return binding.layout == vk::ImageLayout::eGeneral && - info.imageView == binding.image_view && - info.imageLayout == binding.layout; - }) && - null_image.backing.state.layout == vk::ImageLayout::eGeneral, - "shared sampled/storage null descriptors did not retain " - "the general layout through descriptor generation"); - - constexpr uint64_t target_mip_size = 0x10000; - const auto make_target_desc = [&](uint64_t address, uint64_t size, - vk::Extent3D extent) { - ImageDesc desc {}; - desc.type = BindingType::RenderTarget; - desc.info.data = {address, size}; - desc.info.pixel_format = vk::Format::eR32Uint; - desc.info.guest_format = Prospero::BufferFormat::k32UInt; - desc.info.type = Prospero::ImageType::kColor2D; - desc.info.extent = extent; - desc.info.resources = {1, 1}; - desc.info.pitch = extent.width; - desc.info.bytes_per_block = 4; - desc.info.samples = 1; - desc.info.tile_mode = linear; - desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; - desc.view_info.format = desc.info.pixel_format; - desc.view_info.type = vk::ImageViewType::e2D; - desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; - desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - return desc; - }; - - TextureCacheTestAccess::SetLinearReadback(texture_cache, true); - ShaderTextureResource storage {}; - const uint64_t storage_address = base + 0x60000; - constexpr uint32_t storage_native_value = 0x13579bdfu; - constexpr uint32_t storage_stale_value = 0x2468ace0u; - std::memcpy(reinterpret_cast(mapped) + 0x60000, &storage_native_value, - sizeof(storage_native_value)); - const uint64_t encoded_storage_address = storage_address >> 8u; - storage.fields[0] = static_cast(encoded_storage_address); - storage.fields[1] = - static_cast(encoded_storage_address >> 32u) | - (static_cast(stencil_format) << 20u); - storage.fields[3] = DstSel(4, 5, 6, 7) | - (static_cast(linear) << 20u) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - storage.fields[5] = 0x00700000u; - ShaderRecompiler::IR::Program storage_program {}; - storage_program.stage = ShaderType::Vertex; - storage_program.resource_tracking_complete = true; - ShaderRecompiler::IR::ImageResource storage_resource {}; - storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - storage_resource.written = true; - storage_program.info.images.push_back(storage_resource); - ShaderRecompiler::IR::DescriptorValue storage_descriptor {}; - std::copy(std::begin(storage.fields), std::end(storage.fields), - storage_descriptor.dwords.begin()); - storage_descriptor.dword_count = 8; - auto srgb_storage = storage; - constexpr auto srgb_format = - static_cast(Prospero::BufferFormat::k8_8_8_8Srgb); - constexpr uint64_t srgb_storage_address = base + 0xc0000; - const auto encoded_srgb_storage_address = srgb_storage_address >> 8u; - srgb_storage.fields[0] = static_cast(encoded_srgb_storage_address); - srgb_storage.fields[1] = - static_cast(encoded_srgb_storage_address >> 32u) | (srgb_format << 20u); - ShaderRecompiler::IR::DescriptorValue srgb_storage_descriptor {}; - std::copy(std::begin(srgb_storage.fields), std::end(srgb_storage.fields), - srgb_storage_descriptor.dwords.begin()); - srgb_storage_descriptor.dword_count = 8; - auto srgb_storage_resource = storage_resource; - srgb_storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; - const auto srgb_storage_binding = RenderExecutorTestAccess::ResolveTexture( - executor, srgb_storage_resource, srgb_storage_descriptor); - const auto srgb_storage_view = - texture_cache.FindTexture(srgb_storage_binding.image_id, srgb_storage_binding.desc); - Require(name, "sRGB storage view", - srgb_storage_view != nullptr && - srgb_storage_binding.desc.info.pixel_format == vk::Format::eR8G8B8A8Srgb && - srgb_storage_binding.desc.view_info.format == vk::Format::eR8G8B8A8Unorm && - texture_cache.GetImage(srgb_storage_binding.image_id).backing.format == - vk::Format::eR8G8B8A8Srgb, - "storage descriptor did not preserve its sRGB backing and " - "select an UNORM Vulkan view"); - - ShaderTextureResource sint_storage {{0x01514b00u, 0xc1500000u, 0x000bc00bu, 0x91b00204u, - 0x00000000u, 0x00700000u, 0x102b0000u, - 0x0001514au}}; - Require(name, "PPSA06888 R32 SINT descriptor", - sint_storage.Base40() == 0x1514b0000ull && sint_storage.Width5() + 1u == 48 && - sint_storage.Height5() + 1u == 48 && sint_storage.Depth() + 1u == 1 && - sint_storage.Format() == - Prospero::BufferFormat::k32SInt && - sint_storage.Type() == - Prospero::ImageType::kColor2D && - sint_storage.TileMode() == - Prospero::TileMode::kRenderTarget && - sint_storage.DstSelXYZW() == DstSel(4, 0, 0, 1), - "captured write-only signed storage descriptor was decoded " - "incorrectly"); - const uint64_t mapped_sint_address = base + 0xe0000; - const auto encoded_sint_address = mapped_sint_address >> 8u; - sint_storage.fields[0] = static_cast(encoded_sint_address); - sint_storage.fields[1] = (sint_storage.fields[1] & ~0xffu) | - static_cast(encoded_sint_address >> 32u); - ShaderRecompiler::IR::DescriptorValue sint_storage_descriptor {}; - std::copy(std::begin(sint_storage.fields), std::end(sint_storage.fields), - sint_storage_descriptor.dwords.begin()); - sint_storage_descriptor.dword_count = 8; - auto sint_storage_resource = srgb_storage_resource; - sint_storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - const auto sint_storage_binding = RenderExecutorTestAccess::ResolveTexture( - executor, sint_storage_resource, sint_storage_descriptor); - const auto sint_storage_view = - texture_cache.FindTexture(sint_storage_binding.image_id, sint_storage_binding.desc); - Require(name, "PPSA06888 raw R32 SINT storage view", - sint_storage_view != nullptr && - sint_storage_binding.desc.info.data.size == 0x10000 && - sint_storage_binding.desc.info.pixel_format == vk::Format::eR32Sint && - sint_storage_binding.desc.view_info.format == vk::Format::eR32Uint && - texture_cache.GetImage(sint_storage_binding.image_id).backing.format == - vk::Format::eR32Sint, - "write-only R32 SINT storage did not select a bit-compatible " - "uint view"); - - auto narrowed_storage = storage; - constexpr uint64_t narrowed_storage_address = base + 0xd0000; - const auto encoded_narrowed_address = narrowed_storage_address >> 8u; - narrowed_storage.fields[0] = static_cast(encoded_narrowed_address); - narrowed_storage.fields[1] = - static_cast(encoded_narrowed_address >> 32u) | - (static_cast(stencil_format) << 20u); - narrowed_storage.fields[3] = - DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | - (static_cast(Prospero::ImageType::kColor1DArray) << 28u); - narrowed_storage.fields[4] = 1u | (1u << 16u); - ShaderRecompiler::IR::DescriptorValue narrowed_storage_descriptor {}; - std::copy(std::begin(narrowed_storage.fields), std::end(narrowed_storage.fields), - narrowed_storage_descriptor.dwords.begin()); - narrowed_storage_descriptor.dword_count = 8; - auto narrowed_storage_resource = storage_resource; - narrowed_storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim1D; - const auto narrowed_storage_binding = RenderExecutorTestAccess::ResolveTexture( - executor, narrowed_storage_resource, narrowed_storage_descriptor); - const auto narrowed_storage_view = texture_cache.FindTexture( - narrowed_storage_binding.image_id, narrowed_storage_binding.desc); - const auto& narrowed_storage_image = - texture_cache.GetImage(narrowed_storage_binding.image_id); - Require(name, "narrowed 1D array storage view", - narrowed_storage_view != nullptr && - narrowed_storage_binding.desc.info.type == Prospero::ImageType::kColor1D && - narrowed_storage_binding.desc.info.resources.layers == 2 && - narrowed_storage_binding.desc.view_info.type == vk::ImageViewType::e1D && - narrowed_storage_binding.desc.view_info.base_layer == 1 && - narrowed_storage_binding.desc.view_info.layer_count == 1 && - narrowed_storage_image.backing.image_type == vk::ImageType::e1D, - "non-array 1D specialization did not select the descriptor " - "base layer from its 1D-array backing"); - - ShaderRecompiler::IR::ResourceSnapshot storage_snapshot {}; - storage_snapshot.images.push_back(storage_descriptor); - ShaderStageRuntime storage_runtime { - std::make_shared(std::move(storage_program)), - std::make_shared( - std::move(storage_snapshot))}; - auto storage_discovery = executor.PrepareBindings(scheduler.Current(), storage_runtime, - vk::ShaderStageFlagBits::eVertex, - DescriptorCache::Stage::Vertex); - const auto storage_id = storage_discovery.resources.images[0].image_id; - Require(name, "storage prefetch purity", - storage_discovery.resources.images[0].image_view == nullptr && - texture_cache.GetImage(storage_id).binding.is_bound && - !texture_cache.GetImage(storage_id).usage.storage && - !TextureCacheTestAccess::PendingDownload(texture_cache, storage_id), - "the first descriptor pass performed final storage acquisition"); - RenderExecutorTestAccess::ResetBindings(executor); - - ShaderRecompiler::IR::Program sampled_program {}; - sampled_program.stage = ShaderType::Pixel; - sampled_program.resource_tracking_complete = true; - ShaderRecompiler::IR::ImageResource sampled_resource {}; - sampled_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; - sampled_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - sampled_resource.read = true; - sampled_program.info.images.push_back(sampled_resource); - ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot {}; - sampled_snapshot.images.push_back(storage_descriptor); - ShaderStageRuntime sampled_runtime { - std::make_shared(std::move(sampled_program)), - std::make_shared( - std::move(sampled_snapshot))}; - - constexpr uint64_t ordered_sampled_address = base + 0x10000; - const uint32_t ordered_sampled_value = 0x89abcdefu; - std::memcpy(reinterpret_cast(mapped) + 0x10000, &ordered_sampled_value, - sizeof(ordered_sampled_value)); - auto ordered_sampled = storage; - const auto encoded_ordered_address = ordered_sampled_address >> 8u; - ordered_sampled.fields[0] = static_cast(encoded_ordered_address); - ordered_sampled.fields[1] = - static_cast(encoded_ordered_address >> 32u) | - (static_cast(stencil_format) << 20u); - ShaderRecompiler::IR::DescriptorValue ordered_descriptor {}; - std::copy(std::begin(ordered_sampled.fields), std::end(ordered_sampled.fields), - ordered_descriptor.dwords.begin()); - ordered_descriptor.dword_count = 8; - ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot {}; - ordered_snapshot.images.push_back(ordered_descriptor); - ShaderStageRuntime ordered_sampled_runtime { - sampled_runtime.program, - std::make_shared( - std::move(ordered_snapshot))}; - auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( - executor, scheduler.Current(), storage_runtime, ordered_sampled_runtime, true); - const auto ordered_sampled_id = ordered_bindings.pixel->resources.images[0].image_id; - Require(name, "VS-before-PS retained-owner order", - ordered_bindings.vertex.resources.images[0].image_id == storage_id && - ordered_sampled_id != storage_id && - RenderExecutorTestAccess::BoundImagesInOrder( - executor, TextureCacheTestAccess::Owner(texture_cache, storage_id), - TextureCacheTestAccess::Owner(texture_cache, ordered_sampled_id)), - "production graphics binding did not retain vertex resources " - "before pixel resources"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto graphics_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( - executor, scheduler.Current(), storage_runtime, sampled_runtime, true); - const auto& storage_binding = graphics_bindings.vertex.resources.images[0]; - const auto& sampled_binding = graphics_bindings.pixel->resources.images[0]; - Require(name, "storage final acquisition", - storage_binding.image_view != nullptr && - texture_cache.GetImage(storage_id).usage.storage && - texture_cache.GetImage(storage_id).IsGpuModified() && - TextureCacheTestAccess::PendingDownload(texture_cache, storage_id), - "the production graphics binding path did not establish GPU " - "ownership before storage readback"); - Require(name, "VS-to-PS image acquisition order", - storage_binding.image_id == storage_id && - sampled_binding.image_id == storage_id && - texture_cache.GetImage(storage_id).IsGpuModified() && - texture_cache.GetImage(storage_id).usage.storage && - texture_cache.GetImage(storage_id).usage.texture, - "the production graphics binding path did not complete vertex " - "storage acquisition before pixel sampling"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto vertex_sampled_program = - std::make_shared(*sampled_runtime.program); - vertex_sampled_program->stage = ShaderType::Vertex; - ShaderStageRuntime vertex_sampled_runtime {std::move(vertex_sampled_program), - sampled_runtime.resources}; - auto pixel_storage_program = - std::make_shared(*storage_runtime.program); - pixel_storage_program->stage = ShaderType::Pixel; - ShaderStageRuntime pixel_storage_runtime {std::move(pixel_storage_program), - storage_runtime.resources}; - auto writable_alias_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( - executor, scheduler.Current(), vertex_sampled_runtime, pixel_storage_runtime, true); - Require(name, "late writable alias promotion", - texture_cache.GetImage(storage_id).binding.force_general, - "an already sampled image was not promoted when a later " - "storage alias bound the same backing"); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - writable_alias_bindings.vertex); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), - *writable_alias_bindings.pixel); - Require(name, "forced-general descriptor capture", - writable_alias_bindings.vertex.resources.images[0].layout == - vk::ImageLayout::eGeneral && - writable_alias_bindings.pixel->resources.images[0].layout == - vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - writable_alias_bindings.vertex.resources.images[0]) - .imageLayout == vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo( - writable_alias_bindings.pixel->resources.images[0]) - .imageLayout == vk::ImageLayout::eGeneral && - texture_cache.GetImage(storage_id).backing.state.layout == - vk::ImageLayout::eGeneral, - "sampled/storage aliases did not retain the promoted general " - "layout in both generated descriptors"); - RenderExecutorTestAccess::ResetBindings(executor); - - const std::array split_mip_data {0x01020304u, 0x05060708u, 0x090a0b0cu}; - std::memcpy(reinterpret_cast(mapped) + 0x50000, split_mip_data.data(), - sizeof(split_mip_data)); - ImageDesc split_desc {}; - split_desc.type = BindingType::Texture; - split_desc.info.data = {base + 0x50000, sizeof(split_mip_data)}; - split_desc.info.pixel_format = vk::Format::eR32Uint; - split_desc.info.guest_format = Prospero::BufferFormat::k32UInt; - split_desc.info.type = Prospero::ImageType::kColor2D; - split_desc.info.extent = {2, 1, 1}; - split_desc.info.resources = {2, 1}; - split_desc.info.pitch = 2; - split_desc.info.bytes_per_block = 4; - split_desc.info.samples = 1; - split_desc.info.tile_mode = linear; - split_desc.info.mip_layout[0] = {0, 8, 2, 1}; - split_desc.info.mip_layout[1] = {8, 4, 1, 1}; - split_desc.view_info.format = split_desc.info.pixel_format; - split_desc.view_info.type = vk::ImageViewType::e2D; - split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; - split_desc.view_info.level_count = 2; - split_desc.view_info.layer_count = 1; - split_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto split_id = texture_cache.FindImage(split_desc); - - auto split_storage_desc = split_desc; - split_storage_desc.type = BindingType::Storage; - split_storage_desc.view_info.base_level = 0; - split_storage_desc.view_info.level_count = 1; - split_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; - auto split_sampled_desc = split_desc; - split_sampled_desc.view_info.base_level = 1; - split_sampled_desc.view_info.level_count = 1; - split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; - - auto split_program = std::make_shared(); - split_program->stage = ShaderType::Vertex; - split_program->resource_tracking_complete = true; - split_program->info.images = {storage_resource, sampled_resource}; - DescriptorCache::PreparedBindings split_bindings {}; - split_bindings.program = split_program; - split_bindings.snapshot = std::make_shared(); - split_bindings.shader_stage = vk::ShaderStageFlagBits::eVertex; - split_bindings.stage = DescriptorCache::Stage::Vertex; - split_bindings.resources.images.push_back( - {split_id, texture_cache.FindTexture(split_id, split_storage_desc), - split_storage_desc}); - split_bindings.resources.images.push_back( - {split_id, texture_cache.FindTexture(split_id, split_sampled_desc), - split_sampled_desc}); - RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), split_bindings); - const auto& split_image = texture_cache.GetImage(split_id); - Require(name, "per-binding subresource layouts", - split_bindings.resources.images[0].layout == vk::ImageLayout::eGeneral && - split_bindings.resources.images[1].layout == - vk::ImageLayout::eShaderReadOnlyOptimal && - DescriptorCacheTestAccess::MakeImageInfo(split_bindings.resources.images[0]) - .imageLayout == vk::ImageLayout::eGeneral && - DescriptorCacheTestAccess::MakeImageInfo(split_bindings.resources.images[1]) - .imageLayout == vk::ImageLayout::eShaderReadOnlyOptimal && - split_image.backing.subresource_states.size() == 2 && - split_image.backing.subresource_states[0].layout == - vk::ImageLayout::eGeneral && - split_image.backing.subresource_states[1].layout == - vk::ImageLayout::eShaderReadOnlyOptimal, - "descriptor layouts were queried after a later mip transition " - "instead of being captured at each binding"); - - Libs::LibKernel::Memory::WriteBacking(storage_address, &storage_stale_value, - sizeof(storage_stale_value)); - texture_cache.ProcessDownloadImages(); - Require(name, "storage acquisition download consumption", - !TextureCacheTestAccess::PendingDownload(texture_cache, storage_id), - "submit-time processing retained an acquired storage request"); - RenderExecutorTestAccess::ResetBindings(executor); - scheduler.FinishCurrent(); - scheduler.DrainPriorityOperations(); - uint32_t storage_downloaded_value = 0; - Libs::LibKernel::Memory::TryReadBacking(storage_address, &storage_downloaded_value, - sizeof(storage_downloaded_value)); - const uint32_t storage_expected_value = - (storage_stale_value & 0xffffff00u) | (storage_native_value & 0xffu); - Require(name, "storage acquisition readback window", - storage_downloaded_value == storage_expected_value && - texture_cache.GetImage(storage_id).IsGpuModified(), - "readback enrollment was consumed before final storage " - "ownership became visible"); - - constexpr uint64_t phased_depth_address = base + 0x90000; - constexpr uint64_t phased_stencil_address = base + 0xa0000; - constexpr uint64_t phased_htile_address = base + 0xb0000; - HW::DepthRenderTarget phased_depth_target {}; - phased_depth_target.z_info.format = Prospero::DepthFormat::kZ32F; - phased_depth_target.z_info.texture_compatibility = - Prospero::TextureCompatiblePlaneCompression::kEnable; - phased_depth_target.z_info.htile_acceleration = true; - phased_depth_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax; - phased_depth_target.stencil_info.format = Prospero::StencilFormat::k8UInt; - phased_depth_target.stencil_info.texture_compatibility = - Prospero::TextureCompatibleStencil::kEnable; - phased_depth_target.z_read_base_addr = phased_depth_address; - phased_depth_target.z_write_base_addr = phased_depth_address; - phased_depth_target.stencil_read_base_addr = phased_stencil_address; - phased_depth_target.stencil_write_base_addr = phased_stencil_address; - phased_depth_target.htile_data_base_addr = phased_htile_address; - phased_depth_target.size.valid = true; - registers.SetDepthRenderTarget(phased_depth_target); - HW::DepthControl phased_depth_control {}; - phased_depth_control.z_enable = true; - phased_depth_control.z_write_enable = true; - registers.SetDepthControl(phased_depth_control); - HW::RenderControl phased_render_control {}; - registers.SetRenderControl(phased_render_control); - - RenderDepthInfo phased_depth {}; - RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, 1, scheduler.Current(), - phased_depth); - auto non_texture_compatible_target = phased_depth_target; - non_texture_compatible_target.z_info.texture_compatibility = - Prospero::TextureCompatiblePlaneCompression::kDisable; - non_texture_compatible_target.stencil_info.texture_compatibility = - Prospero::TextureCompatibleStencil::kDisable; - registers.SetDepthRenderTarget(non_texture_compatible_target); - RenderDepthInfo non_texture_compatible_depth {}; - RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, 1, scheduler.Current(), - non_texture_compatible_depth); - Require(name, "depth texture compatibility identity", - phased_depth.image_id && phased_depth.htile && - non_texture_compatible_depth.image_id == phased_depth.image_id && - non_texture_compatible_depth.desc.info.data == - phased_depth.desc.info.data && - non_texture_compatible_depth.desc.info.stencil == - phased_depth.desc.info.stencil && - non_texture_compatible_depth.desc.info.metadata.range == - phased_depth.desc.info.metadata.range && - phased_depth.desc.info.metadata.stencil_compressed && - !phased_depth.depth_clear_enable && !phased_depth.depth_meta_clear_enable && - !texture_cache.IsMeta(phased_depth.htile_buffer_vaddr) && - !texture_cache.GetImage(phased_depth.image_id).IsGpuModified() && - !texture_cache.GetImage(phased_depth.image_id).usage.depth_target, - "valid PS5 texture-compatibility policy changed logical depth image " - "identity or discovery side effects"); - registers.SetDepthRenderTarget(phased_depth_target); - RenderColorInfo no_color {}; - auto phased_rendering = RenderExecutorTestAccess::AcquireRenderTargets( - executor, scheduler.Current(), &no_color, 0, phased_depth); - Require( - name, "HTile final acquisition", - phased_depth.image_view != nullptr && phased_rendering.num_color_attachments == 0 && - phased_rendering.depth_stencil_attachment.image_view == - phased_depth.image_view && - phased_rendering.depth_stencil_attachment.has_depth && - phased_depth.depth_meta_clear_enable && phased_depth.depth_load_clear_enable && - texture_cache.IsMeta(phased_depth.htile_buffer_vaddr) && - !texture_cache.IsMetaCleared(phased_depth.htile_buffer_vaddr, - phased_depth.desc.view_info.base_layer) && - texture_cache.GetImage(phased_depth.image_id).IsGpuModified() && - texture_cache.GetImage(phased_depth.image_id).usage.depth_target, - "final depth acquisition did not publish and consume HTile " - "state at the final acquisition boundary"); - RenderExecutorTestAccess::ResetBindings(executor); - - constexpr uint64_t depth_only_address = base + 0x140000; - HW::DepthRenderTarget depth_only_target {}; - depth_only_target.z_info.format = Prospero::DepthFormat::kZ32F; - depth_only_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax; - depth_only_target.stencil_info.htile_stencil_disabled = true; - depth_only_target.z_read_base_addr = depth_only_address; - depth_only_target.z_write_base_addr = depth_only_address; - depth_only_target.size = {63, 63, true}; - registers.SetDepthRenderTarget(depth_only_target); - HW::DepthControl depth_only_control {}; - depth_only_control.stencil_enable = true; - depth_only_control.z_enable = true; - depth_only_control.z_write_enable = true; - depth_only_control.zfunc = static_cast(vk::CompareOp::eAlways); - registers.SetDepthControl(depth_only_control); - HW::RenderControl depth_only_render_control {}; - depth_only_render_control.depth_clear_enable = true; - depth_only_render_control.stencil_clear_enable = true; - registers.SetRenderControl(depth_only_render_control); - RenderDepthInfo depth_only {}; - RenderExecutorTestAccess::ResolveRenderDepthTarget(executor, 1, scheduler.Current(), - depth_only); - Require( - name, "depth-only target with stale stencil state", - depth_only.image_id && depth_only.format == vk::Format::eD32Sfloat && - depth_only.depth_test_enable && depth_only.depth_write_enable && - depth_only.depth_clear_enable && !depth_only.stencil_test_enable && - !depth_only.stencil_clear_enable && depth_only.stencil_buffer_vaddr == 0 && - depth_only.stencil_buffer_size == 0 && depth_only.desc.info.stencil.Empty() && - depth_only.depth_buffer_size != 0 && - depth_only_address + depth_only.depth_buffer_size <= base + allocation_size && - depth_only.vaddr_num == 1 && - depth_only.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, - "raw stencil test or clear state leaked into a depth-only attachment"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto video_subresource = make_target_desc(base + 0x20000, target_mip_size, {1, 1, 1}); - video_subresource.type = BindingType::VideoOut; - video_subresource.info.pixel_format = vk::Format::eR8G8B8A8Srgb; - video_subresource.info.guest_format = - Prospero::BufferFormat::k8_8_8_8Srgb; - video_subresource.view_info.format = video_subresource.info.pixel_format; - video_subresource.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto video_subresource_id = texture_cache.FindImage(video_subresource); - const auto video_subresource_owner = - TextureCacheTestAccess::Owner(texture_cache, video_subresource_id); - auto video_parent = make_target_desc(base + 0x20000, target_mip_size * 2, {2, 2, 1}); - video_parent.type = BindingType::VideoOut; - video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb; - video_parent.info.guest_format = - Prospero::BufferFormat::k8_8_8_8Srgb; - video_parent.info.resources.levels = 2; - video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; - video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, 1}; - video_parent.view_info.format = video_parent.info.pixel_format; - video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled; - const auto video_parent_id = texture_cache.FindImage(video_parent); - const auto video_parent_owner = - TextureCacheTestAccess::Owner(texture_cache, video_parent_id); - Require(name, "video-out replacement remains cache-only", - video_subresource_owner != nullptr && video_parent_owner != nullptr && - video_parent_id != video_subresource_id && - !video_subresource_owner->binding.is_bound && - !video_subresource_owner->binding.is_target && - !video_subresource_owner->binding.needs_rebind && - !video_subresource_owner->usage.video_out && - !video_parent_owner->binding.is_bound && - !video_parent_owner->binding.is_target && - !video_parent_owner->binding.needs_rebind && - !video_parent_owner->usage.video_out, - "FindImage leaked caller-owned state across a video-out " - "replacement"); - - constexpr uint64_t array_target_address = base + 0xd0000; - auto array_target = - make_target_desc(array_target_address, target_mip_size, {128, 128, 1}); - const auto array_target_id = texture_cache.FindImage(array_target); - const auto array_target_owner = - TextureCacheTestAccess::Owner(texture_cache, array_target_id); - RenderExecutorTestAccess::BindRenderTarget(executor, array_target_id); - const auto array_target_view = - texture_cache.FindRenderTarget(array_target_id, array_target); - - ShaderTextureResource array_texture {}; - const auto encoded_array_address = array_target_address >> 8u; - constexpr auto array_format = static_cast(Prospero::BufferFormat::k32UInt); - array_texture.fields[0] = static_cast(encoded_array_address); - array_texture.fields[1] = static_cast(encoded_array_address >> 32u) | - (array_format << 20u) | (3u << 30u); - array_texture.fields[2] = (127u >> 2u) | (127u << 14u); - array_texture.fields[3] = - DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | - (static_cast(Prospero::ImageType::kColor2DArray) << 28u); - array_texture.fields[4] = 1; - - ShaderRecompiler::IR::Program array_program {}; - array_program.stage = ShaderType::Compute; - array_program.resource_tracking_complete = true; - ShaderRecompiler::IR::ImageResource array_resource {}; - array_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; - array_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - array_resource.read = true; - array_program.info.images.push_back(array_resource); - ShaderRecompiler::IR::DescriptorValue array_descriptor {}; - std::copy(std::begin(array_texture.fields), std::end(array_texture.fields), - array_descriptor.dwords.begin()); - array_descriptor.dword_count = 8; - ShaderRecompiler::IR::ResourceSnapshot array_snapshot {}; - array_snapshot.images.push_back(array_descriptor); - ShaderStageRuntime array_runtime { - std::make_shared(std::move(array_program)), - std::make_shared( - std::move(array_snapshot))}; - - auto array_binding = executor.PrepareBindings(scheduler.Current(), array_runtime, - vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), array_binding); - const auto expanded_array_id = array_binding.resources.images[0].image_id; - const auto& expanded_array = texture_cache.GetImage(expanded_array_id); - Require(name, "2D target to array backing expansion", - array_target_view != nullptr && array_target_owner != nullptr && - array_target_owner->binding.needs_rebind && - expanded_array_id != array_target_id && - array_binding.resources.images[0].image_view != nullptr && - array_binding.resources.images[0].desc.info.type == - Prospero::ImageType::kColor2D && - array_binding.resources.images[0].desc.info.resources == - ImageSubresources {1, 2} && - array_binding.resources.images[0].desc.view_info.type == - vk::ImageViewType::e2DArray && - array_binding.resources.images[0].desc.view_info.layer_count == 2 && - expanded_array.backing.layers == 2 && expanded_array.IsGpuModified() && - expanded_array.usage.texture, - "raw array view did not expand and reuse the Color2D backing"); - - RenderColorInfo rebound_array_target {}; - rebound_array_target.type = RenderColorType::RenderTexture; - rebound_array_target.desc = array_target; - rebound_array_target.image_id = array_target_id; - rebound_array_target.format = array_target.view_info.format; - rebound_array_target.extent = {128, 128}; - rebound_array_target.samples = 1; - RenderDepthInfo no_array_depth {}; - auto array_rendering = RenderExecutorTestAccess::AcquireRenderTargets( - executor, scheduler.Current(), &rebound_array_target, 1, no_array_depth); - Require(name, "expanded array target rebind", - rebound_array_target.image_id == expanded_array_id && - rebound_array_target.image_view != nullptr && - array_rendering.num_color_attachments == 1 && - array_rendering.color_attachments[0].image_view == - rebound_array_target.image_view && - array_rendering.width == 128 && array_rendering.height == 128 && - array_rendering.num_layers == 1 && - texture_cache.GetImage(expanded_array_id).binding.is_bound && - texture_cache.GetImage(expanded_array_id).binding.is_target, - "the retained single-layer target did not rebind to the expanded " - "backing"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto colliding_msaa_texture = array_texture; - colliding_msaa_texture.fields[3] = - DstSel(4, 5, 6, 7) | (1u << 16u) | - (static_cast(Prospero::TileMode::kRenderTarget) << 20u) | - (static_cast(Prospero::ImageType::kColor2DMsaa) << 28u); - colliding_msaa_texture.fields[4] = 0; - colliding_msaa_texture.fields[5] = (7u << 20u) | (1u << 4u); - ShaderRecompiler::IR::DescriptorValue colliding_msaa_descriptor {}; - std::copy(std::begin(colliding_msaa_texture.fields), - std::end(colliding_msaa_texture.fields), - colliding_msaa_descriptor.dwords.begin()); - colliding_msaa_descriptor.dword_count = 8; - auto colliding_msaa_program = - std::make_shared(*array_runtime.program); - colliding_msaa_program->info.images[0].dimension = - ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; - auto colliding_msaa_snapshot = - std::make_shared(); - colliding_msaa_snapshot->images.push_back(colliding_msaa_descriptor); - ShaderStageRuntime colliding_msaa_runtime {std::move(colliding_msaa_program), - std::move(colliding_msaa_snapshot)}; - auto colliding_msaa_binding = executor.PrepareBindings( - scheduler.Current(), colliding_msaa_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), colliding_msaa_binding); - const auto& resolved_colliding_msaa = colliding_msaa_binding.resources.images[0]; - Require(name, "equal-footprint sample-count identity", - resolved_colliding_msaa.image_id != expanded_array_id && - resolved_colliding_msaa.image_view != nullptr && - resolved_colliding_msaa.desc.info.type == Prospero::ImageType::kColor2D && - resolved_colliding_msaa.desc.info.resources == ImageSubresources {1, 1} && - resolved_colliding_msaa.desc.info.samples == 2 && - resolved_colliding_msaa.desc.info.data.size == 0x20000 && - texture_cache.GetImage(resolved_colliding_msaa.image_id).backing.samples == - 2 && - texture_cache.GetImage(expanded_array_id).backing.samples == 1 && - texture_cache.GetImage(expanded_array_id).backing.layers == 2, - "equal-size 1x array and 2x image shared an incompatible backing"); - RenderExecutorTestAccess::ResetBindings(executor); - - constexpr uint64_t msaa_target_address = base + 0x100000; - constexpr uint32_t msaa_samples = 4; - auto msaa_target = make_target_desc(msaa_target_address, 0x40000, {128, 128, 1}); - msaa_target.info.samples = msaa_samples; - msaa_target.info.tile_mode = Prospero::TileMode::kRenderTarget; - msaa_target.info.mip_layout[0].size = msaa_target.info.data.size; - const auto msaa_target_id = texture_cache.FindImage(msaa_target); - RenderExecutorTestAccess::BindRenderTarget(executor, msaa_target_id); - const auto msaa_target_view = - texture_cache.FindRenderTarget(msaa_target_id, msaa_target); - - auto msaa_texture = array_texture; - const auto encoded_msaa_address = msaa_target_address >> 8u; - msaa_texture.fields[0] = static_cast(encoded_msaa_address); - msaa_texture.fields[1] = static_cast(encoded_msaa_address >> 32u) | - (array_format << 20u) | (3u << 30u); - msaa_texture.fields[3] = - DstSel(4, 5, 6, 7) | (2u << 16u) | - (static_cast(Prospero::TileMode::kRenderTarget) << 20u) | - (static_cast(Prospero::ImageType::kColor2DMsaa) << 28u); - msaa_texture.fields[4] = 0; - msaa_texture.fields[5] = (7u << 20u) | (2u << 4u); - ShaderRecompiler::IR::DescriptorValue msaa_descriptor {}; - std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields), - msaa_descriptor.dwords.begin()); - msaa_descriptor.dword_count = 8; - auto msaa_program = - std::make_shared(*array_runtime.program); - msaa_program->info.images[0].dimension = - ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; - auto msaa_snapshot = std::make_shared(); - msaa_snapshot->images.push_back(msaa_descriptor); - ShaderStageRuntime msaa_runtime {std::move(msaa_program), std::move(msaa_snapshot)}; - auto msaa_binding = executor.PrepareBindings(scheduler.Current(), msaa_runtime, - vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), msaa_binding); - const auto& resolved_msaa = msaa_binding.resources.images[0]; - Require(name, "MSAA descriptor backing reuse", - msaa_target_view != nullptr && resolved_msaa.image_id == msaa_target_id && - resolved_msaa.image_view != nullptr && - resolved_msaa.desc.info.type == Prospero::ImageType::kColor2D && - resolved_msaa.desc.info.resources == ImageSubresources {1, 1} && - resolved_msaa.desc.info.samples == msaa_samples && - resolved_msaa.desc.info.data.size == 0x40000 && - resolved_msaa.desc.view_info.level_count == 1 && - texture_cache.GetImage(msaa_target_id).backing.samples == msaa_samples, - "MSAA view fields did not resolve to the matching multisample " - "backing"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto msaa_array_texture = msaa_texture; - msaa_array_texture.fields[3] = - (msaa_array_texture.fields[3] & ~(0xfu << 28u)) | - (static_cast(Prospero::ImageType::kColor2DMsaaArray) << 28u); - msaa_array_texture.fields[4] = 1; - ShaderRecompiler::IR::DescriptorValue msaa_array_descriptor {}; - std::copy(std::begin(msaa_array_texture.fields), std::end(msaa_array_texture.fields), - msaa_array_descriptor.dwords.begin()); - msaa_array_descriptor.dword_count = 8; - auto msaa_array_program = - std::make_shared(*msaa_runtime.program); - msaa_array_program->info.images[0].dimension = - ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray; - auto msaa_array_snapshot = std::make_shared(); - msaa_array_snapshot->images.push_back(msaa_array_descriptor); - ShaderStageRuntime msaa_array_runtime {std::move(msaa_array_program), - std::move(msaa_array_snapshot)}; - auto msaa_array_binding = executor.PrepareBindings( - scheduler.Current(), msaa_array_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), msaa_array_binding); - const auto& resolved_msaa_array = msaa_array_binding.resources.images[0]; - Require(name, "MSAA array backing expansion", - resolved_msaa_array.image_id != msaa_target_id && - resolved_msaa_array.image_view != nullptr && - resolved_msaa_array.desc.info.type == Prospero::ImageType::kColor2D && - resolved_msaa_array.desc.info.resources == ImageSubresources {1, 2} && - resolved_msaa_array.desc.info.samples == msaa_samples && - resolved_msaa_array.desc.info.data.size == 0x80000 && - resolved_msaa_array.desc.view_info.type == vk::ImageViewType::e2DArray && - resolved_msaa_array.desc.view_info.layer_count == 2 && - texture_cache.GetImage(resolved_msaa_array.image_id).backing.layers == 2 && - texture_cache.GetImage(resolved_msaa_array.image_id).backing.samples == - msaa_samples, - "MSAA array view did not expand the matching Color2D backing"); - RenderExecutorTestAccess::ResetBindings(executor); - - ImageDesc depth {}; - depth.type = BindingType::DepthTarget; - depth.info.data = {depth_address, 0x10000}; - depth.info.stencil = {stencil_address, stencil_layout.size}; - depth.info.pixel_format = vk::Format::eD32SfloatS8Uint; - depth.info.guest_format = Prospero::BufferFormat::k32Float; - depth.info.type = Prospero::ImageType::kColor2D; - depth.info.extent = {1, 1, 1}; - depth.info.resources = {1, 1}; - depth.info.pitch = 1; - depth.info.bytes_per_block = 4; - depth.info.samples = 1; - depth.info.tile_mode = linear; - depth.info.mip_layout[0] = {0, depth.info.data.size, 1, 1}; - depth.view_info.format = depth.info.pixel_format; - depth.view_info.type = vk::ImageViewType::e2D; - depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; - depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; - const auto depth_id = texture_cache.FindImage(depth); - Require( - name, "cache-only depth discovery", - !texture_cache.FindImageFromRange(stencil_address, stencil_layout.size, false) && - !texture_cache.GetImage(depth_id).binding.is_bound && - !texture_cache.GetImage(depth_id).binding.is_target && - !texture_cache.GetImage(depth_id).usage.depth_target, - "FindImage created depth acquisition state or a stencil proxy"); - RenderExecutorTestAccess::BindRenderTarget(executor, depth_id); - Require( - name, "depth prefetch purity", - texture_cache.GetImage(depth_id).binding.is_target && - !texture_cache.GetImage(depth_id).usage.depth_target && - !texture_cache.FindImageFromRange(stencil_address, stencil_layout.size, false), - "depth prefetch performed final target acquisition"); - - auto target_parent = make_target_desc(base, target_mip_size * 2, {2, 2, 1}); - target_parent.type = BindingType::RenderTarget; - target_parent.info.resources.levels = 2; - target_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; - target_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, 1}; - target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; - auto target_base_subresource = make_target_desc(base, target_mip_size, {2, 2, 1}); - const auto target_base_subresource_id = - texture_cache.FindImage(target_base_subresource); - auto target_subresource = - make_target_desc(base + target_mip_size, target_mip_size, {1, 1, 1}); - const auto target_subresource_id = texture_cache.FindImage(target_subresource); - auto target_subresource_owner = - TextureCacheTestAccess::Owner(texture_cache, target_subresource_id); - std::weak_ptr retired_target = target_subresource_owner; - const auto& discovered_target = texture_cache.GetImage(target_subresource_id); - Require( - name, "cache-only target discovery", - !discovered_target.binding.is_bound && !discovered_target.binding.is_target && - !discovered_target.binding.needs_rebind && - !discovered_target.usage.render_target && - !TextureCacheTestAccess::PendingDownload(texture_cache, target_subresource_id), - "FindImage claimed RenderExecutor-owned render-target state"); - RenderExecutorTestAccess::BindRenderTarget(executor, target_subresource_id); - Require( - name, "target prefetch purity", - texture_cache.GetImage(target_subresource_id).binding.is_target && - !texture_cache.GetImage(target_subresource_id).usage.render_target && - !TextureCacheTestAccess::PendingDownload(texture_cache, target_subresource_id), - "target prefetch performed final target acquisition"); - const auto target_parent_id = texture_cache.FindImage(target_parent); - Require(name, "active target overlap", - target_parent_id && target_parent_id != target_base_subresource_id && - target_parent_id != target_subresource_id && - target_subresource_owner != nullptr && - target_subresource_owner->binding.needs_rebind && - texture_cache.GetImage(target_parent_id).binding.is_target && - !TextureCacheTestAccess::PendingDownload(texture_cache, target_parent_id), - "target overlap did not transfer target state to the merged " - "owner"); - target_subresource_owner.reset(); - - RenderColorInfo rebound_color {}; - rebound_color.type = RenderColorType::RenderTexture; - rebound_color.desc = target_subresource; - rebound_color.image_id = target_subresource_id; - rebound_color.format = target_subresource.view_info.format; - rebound_color.extent = {1, 1}; - rebound_color.samples = 1; - RenderDepthInfo rebound_depth {}; - rebound_depth.desc = depth; - rebound_depth.format = depth.info.pixel_format; - rebound_depth.width = 1; - rebound_depth.height = 1; - rebound_depth.samples = 1; - rebound_depth.image_id = depth_id; - scheduler.FinishCurrent(); - auto retired_owner = retired_target.lock(); - Require(name, "deferred target slot erasure", - TextureCacheTestAccess::Owner(texture_cache, target_subresource_id) == - nullptr && - retired_owner != nullptr && retired_owner->binding.needs_rebind, - "the displaced target did not survive only through RenderExecutor " - "ownership"); - auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets( - executor, scheduler.Current(), &rebound_color, 1, rebound_depth); - const auto stencil_proxy_id = - texture_cache.FindImageFromRange(stencil_address, stencil_layout.size, false); - Require( - name, "production target rebind", - rebound_color.image_id == target_parent_id && rebound_color.image_view != nullptr && - rebound_depth.image_view != nullptr && - rebound_rendering.num_color_attachments == 1 && - rebound_rendering.color_attachments[0].image_view == rebound_color.image_view && - rebound_rendering.depth_stencil_attachment.image_view == - rebound_depth.image_view && - rebound_rendering.width == 1 && rebound_rendering.height == 1 && - texture_cache.GetImage(target_parent_id).binding.is_target && - texture_cache.GetImage(target_parent_id).usage.render_target && - texture_cache.GetImage(target_parent_id).IsGpuModified() && - texture_cache.GetImage(depth_id).usage.depth_target && - texture_cache.GetImage(depth_id).IsGpuModified() && - TextureCacheTestAccess::PendingDownload(texture_cache, target_parent_id) && - stencil_proxy_id && - texture_cache.GetImage(stencil_proxy_id).depth_id == depth_id, - "the final target pass did not acquire color before depth and " - "build the dynamic render state"); - scheduler.BeginRendering(rebound_rendering); - scheduler.EndRendering(); - RenderExecutorTestAccess::ResetBindings(executor); - Require(name, "draw-scoped target reset", - !retired_owner->binding.is_target && !retired_owner->binding.needs_rebind && - !texture_cache.GetImage(target_parent_id).binding.is_target && - !texture_cache.GetImage(depth_id).binding.is_target, - "RenderExecutor retained target binding state after reset"); - retired_owner.reset(); - Require(name, "RenderExecutor-owned retired lifetime", retired_target.expired(), - "a retired target owner survived after RenderExecutor reset"); - - constexpr uint64_t ordered_color_address = base + 0x80000; - auto ordered_color_desc = - make_target_desc(ordered_color_address, target_mip_size, {1, 1, 1}); - const auto stale_ordered_color = texture_cache.FindImage(ordered_color_desc); - RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color); - resources.UnmapMemory(ordered_color_address, target_mip_size); - resources.MapMemory(ordered_color_address, target_mip_size); - - auto ordered_depth_desc = depth; - ordered_depth_desc.info.stencil = {ordered_color_address, target_mip_size}; - const auto ordered_depth_id = texture_cache.FindImage(ordered_depth_desc); - RenderExecutorTestAccess::BindRenderTarget(executor, ordered_depth_id); - RenderColorInfo ordered_color {}; - ordered_color.type = RenderColorType::RenderTexture; - ordered_color.desc = ordered_color_desc; - ordered_color.image_id = stale_ordered_color; - ordered_color.format = ordered_color_desc.view_info.format; - ordered_color.extent = {1, 1}; - ordered_color.samples = 1; - RenderDepthInfo ordered_depth {}; - ordered_depth.desc = ordered_depth_desc; - ordered_depth.format = ordered_depth_desc.info.pixel_format; - ordered_depth.width = 1; - ordered_depth.height = 1; - ordered_depth.samples = 1; - ordered_depth.image_id = ordered_depth_id; - auto ordered_rendering = RenderExecutorTestAccess::AcquireRenderTargets( - executor, scheduler.Current(), &ordered_color, 1, ordered_depth); - Require( - name, "color-before-depth acquisition order", - ordered_color.image_id != stale_ordered_color && - ordered_color.image_view != nullptr && ordered_depth.image_view != nullptr && - ordered_rendering.color_attachments[0].image_view == ordered_color.image_view && - ordered_rendering.depth_stencil_attachment.image_view == - ordered_depth.image_view && - texture_cache.GetImage(ordered_color.image_id).depth_id == - ordered_depth.image_id, - "depth acquisition ran before the stale color target was " - "recreated and associated"); - RenderExecutorTestAccess::ResetBindings(executor); - TextureCacheTestAccess::SetLinearReadback(texture_cache, false); - - ShaderTextureResource stencil {}; - const uint64_t encoded_address = stencil_address >> 8u; - stencil.fields[0] = static_cast(encoded_address); - stencil.fields[1] = - static_cast(encoded_address >> 32u) | - (static_cast(stencil_format) << 20u); - stencil.fields[3] = DstSel(4, 5, 6, 7) | - (static_cast(linear) << 20u) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - - ShaderRecompiler::IR::Program program {}; - program.stage = ShaderType::Compute; - program.resource_tracking_complete = true; - ShaderRecompiler::IR::ImageResource stencil_resource {}; - stencil_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; - stencil_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - stencil_resource.read = true; - program.info.images.push_back(stencil_resource); - - ShaderRecompiler::IR::ResourceSnapshot snapshot {}; - ShaderRecompiler::IR::DescriptorValue descriptor {}; - std::copy(std::begin(stencil.fields), std::end(stencil.fields), - descriptor.dwords.begin()); - descriptor.dword_count = 8; - snapshot.images.push_back(descriptor); - ShaderStageRuntime runtime { - std::make_shared(std::move(program)), - std::make_shared( - std::move(snapshot))}; - - auto prepared = context.GetRenderExecutor().PrepareBindings( - scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - const auto sampled_stencil_id = prepared.resources.images[0].image_id; - Require(name, "first stencil discovery", - prepared.resources.images.size() == 1 && sampled_stencil_id != depth_id && - prepared.resources.images[0].image_view == nullptr && - texture_cache.GetImage(sampled_stencil_id).binding.is_bound && - !texture_cache.GetImage(stencil_proxy_id).binding.is_bound && - !texture_cache.GetImage(stencil_proxy_id).binding.is_target, - "the first sampled-stencil lookup did not remain an ordinary " - "discovery before final depth acquisition"); - - context.GetRenderExecutor().RebindImages(scheduler.Current(), prepared); - Require(name, "first stencil acquisition", - prepared.resources.images[0].image_id == sampled_stencil_id && - prepared.resources.images[0].image_view != nullptr && - texture_cache.GetImage(sampled_stencil_id).usage.texture, - "the first sampled-stencil image was not finally acquired"); - - RenderExecutorTestAccess::ResetBindings(executor); - RenderDepthInfo reassociated_depth = rebound_depth; - reassociated_depth.image_view = nullptr; - RenderColorInfo no_reassociated_color {}; - (void)RenderExecutorTestAccess::AcquireRenderTargets( - executor, scheduler.Current(), &no_reassociated_color, 0, reassociated_depth); - Require(name, "existing stencil association selection", - texture_cache.GetImage(sampled_stencil_id).depth_id == - reassociated_depth.image_id, - "final depth acquisition did not associate the existing image " - "at the stencil guest address"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto redirected = context.GetRenderExecutor().PrepareBindings( - scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - Require(name, "redirected owner discovery", - redirected.resources.images.size() == 1 && - redirected.resources.images[0].image_id == depth_id && - redirected.resources.images[0].image_view == nullptr && - texture_cache.GetImage(depth_id).binding.is_bound && - !texture_cache.GetImage(sampled_stencil_id).binding.is_bound, - "the established stencil association did not redirect the next " - "discovery to the depth owner"); - context.GetRenderExecutor().RebindImages(scheduler.Current(), redirected); - Require(name, "second-pass stencil acquisition", - redirected.resources.images[0].image_id == depth_id && - redirected.resources.images[0].image_view != nullptr && - texture_cache.GetImage(depth_id).usage.texture, - "RebindImages did not acquire the associated depth owner"); - RenderExecutorTestAccess::ResetBindings(executor); - - auto stencil_storage_program = - std::make_shared(*runtime.program); - stencil_storage_program->info.images[0].kind = - ShaderRecompiler::IR::ResourceKind::StorageImageUint; - stencil_storage_program->info.images[0].read = false; - stencil_storage_program->info.images[0].written = true; - auto storage_stencil = stencil; - storage_stencil.fields[5] = 0x00700000u; - ShaderRecompiler::IR::DescriptorValue stencil_storage_descriptor {}; - std::copy(std::begin(storage_stencil.fields), std::end(storage_stencil.fields), - stencil_storage_descriptor.dwords.begin()); - stencil_storage_descriptor.dword_count = 8; - ShaderRecompiler::IR::ResourceSnapshot stencil_storage_snapshot {}; - stencil_storage_snapshot.images.push_back(stencil_storage_descriptor); - ShaderStageRuntime stencil_storage_runtime { - std::move(stencil_storage_program), - std::make_shared( - std::move(stencil_storage_snapshot))}; - auto storage_redirected = executor.PrepareBindings( - scheduler.Current(), stencil_storage_runtime, vk::ShaderStageFlagBits::eCompute, - DescriptorCache::Stage::Compute); - executor.RebindImages(scheduler.Current(), storage_redirected); - Require(name, "storage stencil acquisition", - storage_redirected.resources.images[0].image_id == depth_id && - storage_redirected.resources.images[0].image_view != nullptr && - texture_cache.GetImage(depth_id).usage.storage, - "storage stencil binding did not acquire the associated depth owner"); - RenderExecutorTestAccess::ResetBindings(executor); - resources.UnmapMemory(base, allocation_size); - scheduler.Finish(); - } - - Require(name, "unmap direct backing", - Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, - "descriptor discovery direct mapping release failed"); - Require( - name, "release direct backing", - Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0, - "descriptor discovery direct-memory allocation release failed"); - std::printf("[host] %-32s ok\n", name); - } - - Buffer CreateStorageBuffer(const char* shader_name, const std::vector& initial, - size_t dword_count) { - Buffer ret; - ret.size = static_cast(std::max(dword_count, 1u) * sizeof(u32)); - - vk::BufferCreateInfo buffer_info {}; - buffer_info.sType = vk::StructureType::eBufferCreateInfo; - buffer_info.size = ret.size; - buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer; - buffer_info.sharingMode = vk::SharingMode::eExclusive; - RequireVk(shader_name, "dispatch", - m_device.createBuffer(&buffer_info, nullptr, &ret.buffer), "vkCreateBuffer"); - - vk::MemoryRequirements req {}; - m_device.getBufferMemoryRequirements(ret.buffer, &req); - u32 memory_type = 0; - ret.coherent = FindMemoryType(req.memoryTypeBits, - vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent, - &memory_type); - if (!ret.coherent) { - Require(shader_name, "dispatch", - FindMemoryType(req.memoryTypeBits, vk::MemoryPropertyFlagBits::eHostVisible, - &memory_type), - "no host-visible memory type for storage buffer"); - } - - vk::MemoryAllocateInfo alloc {}; - alloc.sType = vk::StructureType::eMemoryAllocateInfo; - alloc.allocationSize = req.size; - alloc.memoryTypeIndex = memory_type; - RequireVk(shader_name, "dispatch", m_device.allocateMemory(&alloc, nullptr, &ret.memory), - "vkAllocateMemory"); - RequireVk(shader_name, "dispatch", m_device.bindBufferMemory(ret.buffer, ret.memory, 0), - "vkBindBufferMemory"); - - std::vector contents(dword_count, 0); - for (size_t i = 0; i < initial.size() && i < contents.size(); i++) { - contents[i] = initial[i]; - } - WriteBuffer(shader_name, ret, contents); - return ret; - } - - void DestroyBuffer(Buffer* buffer) { - if (buffer == nullptr) { - return; - } - if (buffer->buffer != nullptr) { - m_device.destroyBuffer(buffer->buffer, nullptr); - buffer->buffer = nullptr; - } - if (buffer->memory != nullptr) { - m_device.freeMemory(buffer->memory, nullptr); - buffer->memory = nullptr; - } - } - - Image CreateImage2D(const char* shader_name, u32 width, u32 height, vk::Format format, - vk::ImageUsageFlags usage, const std::vector& initial, - u32 dwords_per_pixel, vk::ImageLayout final_layout) { - std::vector> mips; - if (!initial.empty()) { - mips.push_back(initial); - } - return CreateImageMips(shader_name, width, height, format, usage, mips, dwords_per_pixel, - final_layout, vk::ImageType::e2D, vk::ImageViewType::e2D, 1); - } - - static u32 MipExtent(u32 value, u32 level) { - for (u32 i = 0; i < level && value > 1u; i++) { - value >>= 1u; - } - return std::max(value, 1u); - } - - static size_t ImageMipDwordCount(u32 width, u32 height, u32 dwords_per_pixel, u32 level, - u32 layers = 1) { - return static_cast(MipExtent(width, level)) * - static_cast(MipExtent(height, level)) * layers * dwords_per_pixel; - } - - Image CreateImageMips(const char* shader_name, u32 width, u32 height, vk::Format format, - vk::ImageUsageFlags usage, - const std::vector>& initial_mips, u32 dwords_per_pixel, - vk::ImageLayout final_layout, vk::ImageType image_type, - vk::ImageViewType view_type, u32 layers, u32 view_base_layer = 0, - u32 view_layers = 0) { - Image ret; - ret.format = format; - ret.width = width; - ret.height = height; - ret.layers = layers; - ret.mip_levels = std::max(static_cast(initial_mips.size()), 1u); - ret.dwords_per_pixel = dwords_per_pixel; - - vk::ImageCreateInfo image_info {}; - image_info.sType = vk::StructureType::eImageCreateInfo; - image_info.imageType = image_type; - image_info.format = format; - image_info.extent.width = width; - image_info.extent.height = height; - image_info.extent.depth = 1; - image_info.mipLevels = ret.mip_levels; - image_info.arrayLayers = layers; - image_info.samples = vk::SampleCountFlagBits::e1; - image_info.tiling = vk::ImageTiling::eOptimal; - image_info.usage = - usage | vk::ImageUsageFlagBits::eTransferDst | vk::ImageUsageFlagBits::eTransferSrc; - image_info.sharingMode = vk::SharingMode::eExclusive; - image_info.initialLayout = vk::ImageLayout::eUndefined; - RequireVk(shader_name, "dispatch", m_device.createImage(&image_info, nullptr, &ret.image), - "vkCreateImage"); - - vk::MemoryRequirements req {}; - m_device.getImageMemoryRequirements(ret.image, &req); - u32 memory_type = 0; - if (!FindMemoryType(req.memoryTypeBits, vk::MemoryPropertyFlagBits::eDeviceLocal, - &memory_type)) { - Require(shader_name, "dispatch", FindMemoryType(req.memoryTypeBits, {}, &memory_type), - "no memory type for image"); - } - - vk::MemoryAllocateInfo alloc {}; - alloc.sType = vk::StructureType::eMemoryAllocateInfo; - alloc.allocationSize = req.size; - alloc.memoryTypeIndex = memory_type; - RequireVk(shader_name, "dispatch", m_device.allocateMemory(&alloc, nullptr, &ret.memory), - "vk::Device::allocateMemory(image)"); - RequireVk(shader_name, "dispatch", m_device.bindImageMemory(ret.image, ret.memory, 0), - "vkBindImageMemory"); - - vk::ImageViewCreateInfo view_info {}; - view_info.sType = vk::StructureType::eImageViewCreateInfo; - view_info.image = ret.image; - view_info.viewType = view_type; - view_info.format = format; - view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - view_info.subresourceRange.baseMipLevel = 0; - view_info.subresourceRange.levelCount = ret.mip_levels; - Require(shader_name, "dispatch", view_base_layer < layers, - "image view base layer is out of bounds"); - if (view_layers == 0) { - view_layers = layers - view_base_layer; - } - Require(shader_name, "dispatch", view_layers <= layers - view_base_layer, - "image view layer count is out of bounds"); - view_info.subresourceRange.baseArrayLayer = view_base_layer; - view_info.subresourceRange.layerCount = view_layers; - RequireVk(shader_name, "dispatch", m_device.createImageView(&view_info, nullptr, &ret.view), - "vkCreateImageView"); - - if (!initial_mips.empty()) { - size_t total_dwords = 0; - for (u32 level = 0; level < ret.mip_levels; level++) { - total_dwords += ImageMipDwordCount(width, height, dwords_per_pixel, level, layers); - } - std::vector contents(total_dwords, 0); - size_t offset = 0; - for (u32 level = 0; level < ret.mip_levels; level++) { - const auto level_dwords = - ImageMipDwordCount(width, height, dwords_per_pixel, level, layers); - const auto& src = initial_mips[level]; - for (size_t i = 0; i < src.size() && i < level_dwords; i++) { - contents[offset + i] = src[i]; - } - offset += level_dwords; - } - auto staging = CreateHostBuffer(shader_name, contents.size() * sizeof(u32), - vk::BufferUsageFlagBits::eTransferSrc, contents); - UploadImageMips(shader_name, &ret, staging.buffer, final_layout); - DestroyBuffer(&staging); - } else { - TransitionImage(shader_name, &ret, final_layout, vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eComputeShader, {}, - AccessForLayout(final_layout)); - } - return ret; - } - - void DestroyImage(Image* image) { - if (image == nullptr) { - return; - } - if (image->view != nullptr) { - m_device.destroyImageView(image->view, nullptr); - image->view = nullptr; - } - if (image->image != nullptr) { - m_device.destroyImage(image->image, nullptr); - image->image = nullptr; - } - if (image->memory != nullptr) { - m_device.freeMemory(image->memory, nullptr); - image->memory = nullptr; - } - } - - std::vector ReadImage(const char* shader_name, Image* image) { - const auto dword_count = static_cast(image->width) * - static_cast(image->height) * image->layers * - image->dwords_per_pixel; - auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32), - vk::BufferUsageFlagBits::eTransferDst, {}); - - vk::CommandBuffer cmd = BeginCommands(shader_name, "readback"); - AddImageBarrier(cmd, image->image, image->layout, vk::ImageLayout::eTransferSrcOptimal, - vk::PipelineStageFlagBits::eAllCommands, - vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, - vk::AccessFlagBits::eTransferRead, image->mip_levels, image->layers); - vk::BufferImageCopy copy {}; - copy.bufferOffset = 0; - copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - copy.imageSubresource.mipLevel = 0; - copy.imageSubresource.baseArrayLayer = 0; - copy.imageSubresource.layerCount = image->layers; - copy.imageExtent.width = image->width; - copy.imageExtent.height = image->height; - copy.imageExtent.depth = 1; - cmd.copyImageToBuffer(image->image, vk::ImageLayout::eTransferSrcOptimal, staging.buffer, 1, - ©); - EndSubmitAndFree(shader_name, "readback", cmd); - image->layout = vk::ImageLayout::eTransferSrcOptimal; - - auto ret = ReadBuffer(shader_name, staging, dword_count); - DestroyBuffer(&staging); - return ret; - } - - vk::Sampler CreateNearestSampler(const char* shader_name) { - vk::SamplerCreateInfo sampler_info {}; - sampler_info.sType = vk::StructureType::eSamplerCreateInfo; - sampler_info.magFilter = vk::Filter::eNearest; - sampler_info.minFilter = vk::Filter::eNearest; - sampler_info.mipmapMode = vk::SamplerMipmapMode::eNearest; - sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge; - sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge; - sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; - sampler_info.minLod = 0.0f; - sampler_info.maxLod = 0.0f; - vk::Sampler sampler = nullptr; - RequireVk(shader_name, "dispatch", m_device.createSampler(&sampler_info, nullptr, &sampler), - "vkCreateSampler"); - return sampler; - } - - std::vector ReadBuffer(const char* shader_name, const Buffer& buffer, size_t dword_count) { - if (!buffer.coherent) { - vk::MappedMemoryRange range {}; - range.sType = vk::StructureType::eMappedMemoryRange; - range.memory = buffer.memory; - range.offset = 0; - range.size = VK_WHOLE_SIZE; - RequireVk(shader_name, "readback", m_device.invalidateMappedMemoryRanges(1, &range), - "vkInvalidateMappedMemoryRanges"); - } - - void* data = nullptr; - RequireVk(shader_name, "readback", - m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data), "vkMapMemory"); - std::vector ret(dword_count, 0); - std::memcpy(ret.data(), data, dword_count * sizeof(u32)); - m_device.unmapMemory(buffer.memory); - return ret; - } - - void Dispatch(const TestCase& test, const CompiledShader& compiled, const Buffer& buffer, - const Buffer* gds_buffer = nullptr, const Image* sampled_image = nullptr, - const Image* storage_image = nullptr, const Image* storage_image_uint = nullptr, - vk::Sampler sampler = nullptr) { - using Kind = ShaderRecompiler::IR::DescriptorBindingKind; - const auto& layout = compiled.program.bindings; - auto Binding = [&](Kind kind) { return ShaderRecompiler::IR::FindBinding(layout, kind); }; - auto Count = [&](Kind kind) { - const auto* binding = Binding(kind); - if (binding == nullptr) { - return 0u; - } - if (kind == Kind::Gds) { - return 1u; - } - return static_cast(binding->resources.size()); - }; - Require(test.name, "dispatch", - Count(Kind::Sampled2DArray) == 0 && Count(Kind::Sampled3D) == 0 && - Count(Kind::SampledUint2DArray) == 0 && Count(Kind::SampledUint3D) == 0 && - Count(Kind::Storage1D) == 0 && Count(Kind::Storage1DArray) == 0 && - Count(Kind::Storage2DArray) == 0 && Count(Kind::Storage3D) == 0 && - Count(Kind::StorageUint1D) == 0 && Count(Kind::StorageUint1DArray) == 0 && - Count(Kind::StorageUint2DArray) == 0 && Count(Kind::StorageUint3D) == 0, - "unsupported array/3D image cases must provide matching Vulkan test " - "views " - "before dispatch"); - - vk::ShaderModuleCreateInfo module_info {}; - module_info.sType = vk::StructureType::eShaderModuleCreateInfo; - module_info.codeSize = compiled.spirv.size() * sizeof(u32); - module_info.pCode = compiled.spirv.data(); - vk::ShaderModule module = nullptr; - RequireVk(test.name, "SPIR-V module", - m_device.createShaderModule(&module_info, nullptr, &module), - "vkCreateShaderModule"); - - std::vector layout_bindings; - auto add_layout_binding = [&layout_bindings](u32 binding, vk::DescriptorType type, - u32 count) { - if (count == 0) { - return; - } - vk::DescriptorSetLayoutBinding item {}; - item.binding = binding; - item.descriptorType = type; - item.descriptorCount = count; - item.stageFlags = vk::ShaderStageFlagBits::eCompute; - layout_bindings.push_back(item); - }; - for (const auto& binding: layout.descriptors) { - vk::DescriptorType type = vk::DescriptorType::eStorageBuffer; - u32 count = static_cast(binding.resources.size()); - switch (binding.kind) { - case Kind::Sampled1D: - case Kind::Sampled1DArray: - case Kind::Sampled2D: - case Kind::Sampled2DArray: - case Kind::Sampled3D: - case Kind::SampledUint1D: - case Kind::SampledUint1DArray: - case Kind::SampledUint2D: - case Kind::SampledUint2DArray: - case Kind::SampledUint3D: type = vk::DescriptorType::eSampledImage; break; - case Kind::Storage1D: - case Kind::Storage1DArray: - case Kind::Storage2D: - case Kind::Storage2DArray: - case Kind::Storage3D: - case Kind::StorageUint1D: - case Kind::StorageUint1DArray: - case Kind::StorageUint2D: - case Kind::StorageUint2DArray: - case Kind::StorageUint3D: type = vk::DescriptorType::eStorageImage; break; - case Kind::Samplers: type = vk::DescriptorType::eSampler; break; - case Kind::Gds: - case Kind::FlattenedSrt: - case Kind::UserData: count = 1; break; - default: break; - } - add_layout_binding(binding.binding, type, count); - } - - vk::DescriptorSetLayoutCreateInfo layout_info {}; - layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; - layout_info.bindingCount = static_cast(layout_bindings.size()); - layout_info.pBindings = layout_bindings.empty() ? nullptr : layout_bindings.data(); - vk::DescriptorSetLayout descriptor_layout = nullptr; - RequireVk(test.name, "dispatch", - m_device.createDescriptorSetLayout(&layout_info, nullptr, &descriptor_layout), - "vkCreateDescriptorSetLayout"); - - vk::PipelineLayoutCreateInfo pipeline_layout_info {}; - pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; - pipeline_layout_info.setLayoutCount = 1; - pipeline_layout_info.pSetLayouts = &descriptor_layout; - vk::PushConstantRange push_range {}; - if (layout.push_constant_size != 0) { - push_range.stageFlags = vk::ShaderStageFlagBits::eCompute; - push_range.offset = layout.push_constant_offset; - push_range.size = layout.push_constant_size; - pipeline_layout_info.pushConstantRangeCount = 1; - pipeline_layout_info.pPushConstantRanges = &push_range; - } - vk::PipelineLayout pipeline_layout = nullptr; - RequireVk(test.name, "dispatch", - m_device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline_layout), - "vkCreatePipelineLayout"); - - vk::PipelineShaderStageCreateInfo stage {}; - stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; - stage.stage = vk::ShaderStageFlagBits::eCompute; - stage.module = module; - stage.pName = "main"; - - vk::ComputePipelineCreateInfo pipeline_info {}; - pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo; - pipeline_info.stage = stage; - pipeline_info.layout = pipeline_layout; - vk::Pipeline pipeline = nullptr; - RequireVk(test.name, "dispatch", - m_device.createComputePipelines(nullptr, 1, &pipeline_info, nullptr, &pipeline), - "vkCreateComputePipelines"); - - std::vector pool_sizes; - auto add_pool_size = [&pool_sizes](vk::DescriptorType type, u32 count) { - if (count == 0) { - return; - } - for (auto& size: pool_sizes) { - if (size.type == type) { - size.descriptorCount += count; - return; - } - } - pool_sizes.push_back({type, count}); - }; - add_pool_size(vk::DescriptorType::eStorageBuffer, - Count(Kind::Buffers) + Count(Kind::AddressMemory) + Count(Kind::Gds) + - (Binding(Kind::FlattenedSrt) != nullptr ? 1u : 0u) + - (Binding(Kind::UserData) != nullptr ? 1u : 0u)); - add_pool_size(vk::DescriptorType::eSampledImage, - Count(Kind::Sampled1D) + Count(Kind::Sampled1DArray) + - Count(Kind::Sampled2D) + Count(Kind::Sampled2DArray) + - Count(Kind::Sampled3D) + Count(Kind::SampledUint1D) + - Count(Kind::SampledUint1DArray) + Count(Kind::SampledUint2D) + - Count(Kind::SampledUint2DArray) + Count(Kind::SampledUint3D)); - add_pool_size(vk::DescriptorType::eStorageImage, - Count(Kind::Storage1D) + Count(Kind::Storage1DArray) + - Count(Kind::Storage2D) + Count(Kind::Storage2DArray) + - Count(Kind::Storage3D) + Count(Kind::StorageUint1D) + - Count(Kind::StorageUint1DArray) + Count(Kind::StorageUint2D) + - Count(Kind::StorageUint2DArray) + Count(Kind::StorageUint3D)); - add_pool_size(vk::DescriptorType::eSampler, Count(Kind::Samplers)); - vk::DescriptorPoolCreateInfo pool_info {}; - pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; - pool_info.maxSets = 1; - pool_info.poolSizeCount = static_cast(pool_sizes.size()); - pool_info.pPoolSizes = pool_sizes.empty() ? nullptr : pool_sizes.data(); - vk::DescriptorPool descriptor_pool = nullptr; - RequireVk(test.name, "dispatch", - m_device.createDescriptorPool(&pool_info, nullptr, &descriptor_pool), - "vkCreateDescriptorPool"); - - vk::DescriptorSetAllocateInfo set_info {}; - set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo; - set_info.descriptorPool = descriptor_pool; - set_info.descriptorSetCount = 1; - set_info.pSetLayouts = &descriptor_layout; - vk::DescriptorSet descriptor_set = nullptr; - RequireVk(test.name, "dispatch", - m_device.allocateDescriptorSets(&set_info, &descriptor_set), - "vkAllocateDescriptorSets"); - - std::vector writes; - std::vector buffer_infos; - std::vector address_memory_infos; - std::vector sampled_infos; - std::vector storage_infos; - std::vector storage_uint_infos; - std::vector sampler_infos; - Buffer flattened_buffer; - Buffer user_data_buffer; - vk::DescriptorBufferInfo flattened_info {}; - vk::DescriptorBufferInfo user_data_info {}; - vk::DescriptorBufferInfo gds_info {}; - - const auto* buffers = Binding(Kind::Buffers); - if (buffers != nullptr) { - buffer_infos.resize(buffers->resources.size()); - for (u32 i = 0; i < buffer_infos.size(); i++) { - auto& info = buffer_infos[i]; - info.buffer = buffer.buffer; - info.offset = 0; - info.range = buffer.size; - if (test.storage_buffer_range_dwords != 0) { - const auto offset = i < test.storage_buffer_offsets.size() - ? test.storage_buffer_offsets[i] - : 0u; - info.range = static_cast( - test.storage_buffer_range_dwords * sizeof(u32) + offset); - Require(test.name, "dispatch", info.range <= buffer.size, - "storage buffer descriptor range exceeds backing buffer"); - } - } - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = buffers->binding; - write.descriptorCount = static_cast(buffer_infos.size()); - write.descriptorType = vk::DescriptorType::eStorageBuffer; - write.pBufferInfo = buffer_infos.data(); - writes.push_back(write); - } - if (const auto* address = Binding(Kind::AddressMemory); address != nullptr) { - address_memory_infos.resize(address->resources.size()); - for (auto& info: address_memory_infos) { - info = {buffer.buffer, 0, buffer.size}; - } - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = address->binding; - write.descriptorCount = static_cast(address_memory_infos.size()); - write.descriptorType = vk::DescriptorType::eStorageBuffer; - write.pBufferInfo = address_memory_infos.data(); - writes.push_back(write); - } - if (const auto* flattened = Binding(Kind::FlattenedSrt); flattened != nullptr) { - flattened_buffer = CreateStorageBuffer(test.name, compiled.flattened_srt, - compiled.flattened_srt.size()); - flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size}; - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = flattened->binding; - write.descriptorCount = 1; - write.descriptorType = vk::DescriptorType::eStorageBuffer; - write.pBufferInfo = &flattened_info; - writes.push_back(write); - } - if (const auto* user = Binding(Kind::UserData); user != nullptr) { - user_data_buffer = CreateStorageBuffer(test.name, compiled.packed_user_data, - compiled.packed_user_data.size()); - user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size}; - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = user->binding; - write.descriptorCount = 1; - write.descriptorType = vk::DescriptorType::eStorageBuffer; - write.pBufferInfo = &user_data_info; - writes.push_back(write); - } - if (const auto* gds = Binding(Kind::Gds); gds != nullptr) { - Require(test.name, "dispatch", gds_buffer != nullptr, - "GDS descriptor requested but no GDS buffer was provided"); - gds_info = {gds_buffer->buffer, 0, gds_buffer->size}; - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = gds->binding; - write.descriptorCount = 1; - write.descriptorType = vk::DescriptorType::eStorageBuffer; - write.pBufferInfo = &gds_info; - writes.push_back(write); - } - const ShaderRecompiler::IR::DescriptorBinding* sampled = nullptr; - constexpr std::array sampled_kinds { - Kind::Sampled1D, Kind::Sampled1DArray, Kind::Sampled2D, - Kind::SampledUint1D, Kind::SampledUint1DArray, Kind::SampledUint2D, - }; - for (const auto kind: sampled_kinds) { - if (const auto* candidate = Binding(kind); candidate != nullptr) { - Require(test.name, "dispatch", sampled == nullptr, - "Vulkan test harness needs separate sampled images for mixed " - "descriptor classes"); - sampled = candidate; - } - } - if (sampled != nullptr) { - Require(test.name, "dispatch", sampled_image != nullptr, - "sampled image descriptor requested but no sampled image was " - "provided"); - sampled_infos.resize(sampled->resources.size()); - for (auto& info: sampled_infos) { - info.imageView = sampled_image->view; - info.imageLayout = sampled_image->layout; - } - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = sampled->binding; - write.descriptorCount = static_cast(sampled_infos.size()); - write.descriptorType = vk::DescriptorType::eSampledImage; - write.pImageInfo = sampled_infos.data(); - writes.push_back(write); - } - const auto* storage = Binding(Kind::Storage2D); - const auto* storage_uint = Binding(Kind::StorageUint2D); - if (storage != nullptr || storage_uint != nullptr) { - Require(test.name, "dispatch", storage_image != nullptr, - "storage image descriptor requested but no storage image was " - "provided"); - Require(test.name, "dispatch", storage_image_uint != nullptr, - "uint storage image descriptor requested but no uint storage " - "image was provided"); - storage_infos.resize(storage != nullptr ? storage->resources.size() : 0u); - storage_uint_infos.resize(storage_uint != nullptr ? storage_uint->resources.size() - : 0u); - for (auto& info: storage_infos) { - info.imageView = storage_image->view; - info.imageLayout = storage_image->layout; - } - for (auto& info: storage_uint_infos) { - info.imageView = storage_image_uint->view; - info.imageLayout = storage_image_uint->layout; - } - if (storage != nullptr) { - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = storage->binding; - write.descriptorCount = static_cast(storage_infos.size()); - write.descriptorType = vk::DescriptorType::eStorageImage; - write.pImageInfo = storage_infos.data(); - writes.push_back(write); - } - if (storage_uint != nullptr) { - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = storage_uint->binding; - write.descriptorCount = static_cast(storage_uint_infos.size()); - write.descriptorType = vk::DescriptorType::eStorageImage; - write.pImageInfo = storage_uint_infos.data(); - writes.push_back(write); - } - } - const auto* samplers = Binding(Kind::Samplers); - if (samplers != nullptr) { - Require(test.name, "dispatch", sampler != nullptr, - "sampler descriptor requested but no sampler was provided"); - sampler_infos.resize(samplers->resources.size()); - for (auto& info: sampler_infos) { - info.sampler = sampler; - } - vk::WriteDescriptorSet write {}; - write.sType = vk::StructureType::eWriteDescriptorSet; - write.dstSet = descriptor_set; - write.dstBinding = samplers->binding; - write.descriptorCount = static_cast(sampler_infos.size()); - write.descriptorType = vk::DescriptorType::eSampler; - write.pImageInfo = sampler_infos.data(); - writes.push_back(write); - } - if (!writes.empty()) { - m_device.updateDescriptorSets(static_cast(writes.size()), writes.data(), 0, - nullptr); - } - - vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch"); - cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline); - cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, - &descriptor_set, 0, nullptr); - if (layout.push_constant_size != 0) { - Require(test.name, "dispatch", - compiled.packed_user_data.size() * sizeof(u32) == layout.push_constant_size, - "native user-data size does not match push-constant range"); - cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, - layout.push_constant_offset, layout.push_constant_size, - compiled.packed_user_data.data()); - } - cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z); - - if (buffers != nullptr) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer.buffer; - barrier.offset = 0; - barrier.size = buffer.size; - cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, - vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, - nullptr); - } - if (gds_buffer != nullptr) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = - vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; - barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = gds_buffer->buffer; - barrier.offset = 0; - barrier.size = gds_buffer->size; - cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, - vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, - nullptr); - } - EndSubmitAndFree(test.name, "dispatch", cmd); - if (flattened_buffer.buffer != nullptr) { - DestroyBuffer(&flattened_buffer); - } - if (user_data_buffer.buffer != nullptr) { - DestroyBuffer(&user_data_buffer); - } - m_device.destroyDescriptorPool(descriptor_pool, nullptr); - m_device.destroyPipeline(pipeline, nullptr); - m_device.destroyPipelineLayout(pipeline_layout, nullptr); - m_device.destroyDescriptorSetLayout(descriptor_layout, nullptr); - m_device.destroyShaderModule(module, nullptr); - } - - std::vector RenderFragment(const GraphicsCase& test, const CompiledShader& fragment) { - const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(); - ValidateSpirv(test.name, vertex_spirv); - - Image target = CreateImage2D(test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat, - vk::ImageUsageFlagBits::eColorAttachment, {}, 4, - vk::ImageLayout::eGeneral); - const std::vector default_vertices = { - 0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u, - 0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u, - 0xbf800000u, 0x40400000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u, - }; - const auto& vertices = test.vertices.empty() ? default_vertices : test.vertices; - Require(test.name, "graphics", vertices.size() == 18u, - "graphics vertex buffer must contain three pos2/color4 vertices"); - auto vertex_buffer = CreateHostBuffer(test.name, vertices.size() * sizeof(u32), - vk::BufferUsageFlagBits::eVertexBuffer, vertices); - - auto make_module = [&](const std::vector& spirv) { - vk::ShaderModuleCreateInfo module_info {}; - module_info.sType = vk::StructureType::eShaderModuleCreateInfo; - module_info.codeSize = spirv.size() * sizeof(u32); - module_info.pCode = spirv.data(); - vk::ShaderModule module = nullptr; - RequireVk(test.name, "graphics", - m_device.createShaderModule(&module_info, nullptr, &module), - "vkCreateShaderModule"); - return module; - }; - vk::ShaderModule vertex_module = make_module(vertex_spirv); - vk::ShaderModule fragment_module = make_module(fragment.spirv); - - const auto& fragment_bind = fragment.program.bindings; - vk::PushConstantRange push_constant_range {}; - if (fragment_bind.push_constant_size > 0) { - Require(test.name, "graphics", - test.push_constants.size() * sizeof(u32) == fragment_bind.push_constant_size, - "fragment push constant data size does not match reflection"); - push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment; - push_constant_range.offset = fragment_bind.push_constant_offset; - push_constant_range.size = fragment_bind.push_constant_size; - } - - vk::PipelineLayoutCreateInfo pipeline_layout_info {}; - pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; - pipeline_layout_info.pushConstantRangeCount = push_constant_range.size != 0 ? 1u : 0u; - pipeline_layout_info.pPushConstantRanges = - push_constant_range.size != 0 ? &push_constant_range : nullptr; - vk::PipelineLayout pipeline_layout = nullptr; - RequireVk(test.name, "graphics", - m_device.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline_layout), - "vkCreatePipelineLayout"); - - vk::PipelineShaderStageCreateInfo stages[2] = {}; - stages[0].sType = vk::StructureType::ePipelineShaderStageCreateInfo; - stages[0].stage = vk::ShaderStageFlagBits::eVertex; - stages[0].module = vertex_module; - stages[0].pName = "main"; - stages[1].sType = vk::StructureType::ePipelineShaderStageCreateInfo; - stages[1].stage = vk::ShaderStageFlagBits::eFragment; - stages[1].module = fragment_module; - stages[1].pName = "main"; - - vk::VertexInputBindingDescription vertex_binding {}; - vertex_binding.binding = 0; - vertex_binding.stride = 6u * sizeof(float); - vertex_binding.inputRate = vk::VertexInputRate::eVertex; - vk::VertexInputAttributeDescription attributes[2] = {}; - attributes[0].location = 0; - attributes[0].binding = 0; - attributes[0].format = vk::Format::eR32G32Sfloat; - attributes[0].offset = 0; - attributes[1].location = 1; - attributes[1].binding = 0; - attributes[1].format = vk::Format::eR32G32B32A32Sfloat; - attributes[1].offset = 2u * sizeof(float); - vk::PipelineVertexInputStateCreateInfo vertex_input {}; - vertex_input.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo; - vertex_input.vertexBindingDescriptionCount = 1; - vertex_input.pVertexBindingDescriptions = &vertex_binding; - vertex_input.vertexAttributeDescriptionCount = 2; - vertex_input.pVertexAttributeDescriptions = attributes; - - vk::PipelineInputAssemblyStateCreateInfo input_assembly {}; - input_assembly.sType = vk::StructureType::ePipelineInputAssemblyStateCreateInfo; - input_assembly.topology = vk::PrimitiveTopology::eTriangleList; - - vk::Viewport viewport {}; - viewport.x = 0.0f; - viewport.y = 0.0f; - viewport.width = 1.0f; - viewport.height = 1.0f; - viewport.minDepth = 0.0f; - viewport.maxDepth = 1.0f; - vk::Rect2D scissor {}; - scissor.extent.width = 1; - scissor.extent.height = 1; - vk::PipelineViewportStateCreateInfo viewport_state {}; - viewport_state.sType = vk::StructureType::ePipelineViewportStateCreateInfo; - viewport_state.viewportCount = 1; - viewport_state.pViewports = &viewport; - viewport_state.scissorCount = 1; - viewport_state.pScissors = &scissor; - - vk::PipelineRasterizationStateCreateInfo raster {}; - raster.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo; - raster.polygonMode = vk::PolygonMode::eFill; - raster.cullMode = vk::CullModeFlagBits::eNone; - raster.frontFace = vk::FrontFace::eCounterClockwise; - raster.lineWidth = 1.0f; - - vk::PipelineMultisampleStateCreateInfo multisample {}; - multisample.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo; - multisample.rasterizationSamples = vk::SampleCountFlagBits::e1; - - vk::PipelineColorBlendAttachmentState color_attachment {}; - color_attachment.colorWriteMask = - vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | - vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA; - vk::PipelineColorBlendStateCreateInfo color_blend {}; - color_blend.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo; - color_blend.attachmentCount = 1; - color_blend.pAttachments = &color_attachment; - - vk::GraphicsPipelineCreateInfo pipeline_info {}; - const vk::Format color_format = vk::Format::eR32G32B32A32Sfloat; - vk::PipelineRenderingCreateInfo rendering_pipeline {}; - rendering_pipeline.sType = vk::StructureType::ePipelineRenderingCreateInfo; - rendering_pipeline.colorAttachmentCount = 1; - rendering_pipeline.pColorAttachmentFormats = &color_format; - pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo; - pipeline_info.pNext = &rendering_pipeline; - pipeline_info.stageCount = 2; - pipeline_info.pStages = stages; - pipeline_info.pVertexInputState = &vertex_input; - pipeline_info.pInputAssemblyState = &input_assembly; - pipeline_info.pViewportState = &viewport_state; - pipeline_info.pRasterizationState = &raster; - pipeline_info.pMultisampleState = &multisample; - pipeline_info.pColorBlendState = &color_blend; - pipeline_info.layout = pipeline_layout; - vk::Pipeline pipeline = nullptr; - RequireVk(test.name, "graphics", - m_device.createGraphicsPipelines(nullptr, 1, &pipeline_info, nullptr, &pipeline), - "vkCreateGraphicsPipelines"); - - vk::CommandBuffer cmd = BeginCommands(test.name, "graphics"); - vk::RenderingAttachmentInfo color {}; - color.sType = vk::StructureType::eRenderingAttachmentInfo; - color.imageView = target.view; - color.imageLayout = vk::ImageLayout::eGeneral; - color.loadOp = vk::AttachmentLoadOp::eClear; - color.storeOp = vk::AttachmentStoreOp::eStore; - vk::RenderingInfo rendering {}; - rendering.sType = vk::StructureType::eRenderingInfo; - rendering.renderArea.extent = {1, 1}; - rendering.layerCount = 1; - rendering.colorAttachmentCount = 1; - rendering.pColorAttachments = &color; - cmd.beginRendering(rendering); - cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); - if (push_constant_range.size != 0) { - cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, - fragment_bind.push_constant_offset, fragment_bind.push_constant_size, - test.push_constants.data()); - } - vk::DeviceSize offset = 0; - cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset); - cmd.draw(3, 1, 0, 0); - cmd.endRendering(); - EndSubmitAndFree(test.name, "graphics", cmd); - target.layout = vk::ImageLayout::eGeneral; - - auto pixel = ReadImage(test.name, &target); - pixel.resize(4); - - m_device.destroyPipeline(pipeline, nullptr); - m_device.destroyPipelineLayout(pipeline_layout, nullptr); - m_device.destroyShaderModule(fragment_module, nullptr); - m_device.destroyShaderModule(vertex_module, nullptr); - DestroyBuffer(&vertex_buffer); - DestroyImage(&target); - return pixel; - } - - void CheckGpuTilerCpuParity() { - constexpr const char* name = "GpuTilerCpuParity"; - EnsureRuntimeContext(); - - CommandScheduler scheduler(Renderer(), m_runtime_context); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - StreamBuffer parameters(m_runtime_context, scheduler, MemoryUsage::Stream, 1u << 20); - TileManager tile_manager(m_runtime_context, scheduler, parameters); - constexpr uint64_t conversion_limit = 128ull << 20; - Require(name, "D16 conversion chunk boundaries", - TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10, 4096, 256, - conversion_limit, 65535) == 4096 && - TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10, 4097, 256, - conversion_limit, 65535) == 4096 && - TileManagerTestAccess::ConversionRows(257, 32ull << 10, 32ull << 10, 4097, 256, - conversion_limit, 65535) == 4095, - "D16 conversion planner exceeds descriptor or dispatch limits"); - - const auto align_dword = [](uint64_t size) { - return (size + sizeof(u32) - 1u) & ~(uint64_t {sizeof(u32) - 1u}); - }; - const auto to_dwords = [&](const std::vector& bytes, uint64_t padded_size) { - std::vector words(static_cast(padded_size / sizeof(u32)), 0); - std::memcpy(words.data(), bytes.data(), bytes.size()); - return words; - }; - const auto read_bytes = [&](const Buffer& buffer, uint64_t size) { - const auto words = - ReadBuffer(name, buffer, static_cast(align_dword(size) / sizeof(u32))); - std::vector bytes(static_cast(size)); - std::memcpy(bytes.data(), words.data(), bytes.size()); - return bytes; - }; - const auto host_barrier = [&](vk::Buffer buffer, uint64_t size, - vk::PipelineStageFlags source_stage, - vk::AccessFlags source_access) { - vk::BufferMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eBufferMemoryBarrier; - barrier.srcAccessMask = source_access; - barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.buffer = buffer; - barrier.offset = 0; - barrier.size = size; - scheduler.Current().Handle().pipelineBarrier(source_stage, - vk::PipelineStageFlagBits::eHost, {}, 0, - nullptr, 1, &barrier, 0, nullptr); - }; - const auto gpu_detile = [&](const std::vector& tiled, std::vector* linear, - uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos) { - const uint64_t padded_tiled = align_dword(tiled_capacity); - const uint64_t padded_linear = align_dword(linear_capacity); - auto input = - CreateHostBuffer(name, padded_tiled, AllFlags, to_dwords(tiled, padded_tiled)); - auto output = CreateHostBuffer( - name, padded_linear, AllFlags, - std::vector(static_cast(padded_linear / sizeof(u32)), 0xabababab)); - const auto result = - tile_manager.Detile(input.buffer, 0, padded_tiled, padded_linear, infos); - const vk::BufferCopy copy {result.offset, 0, padded_linear}; - scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer, 1, ©); - host_barrier(output.buffer, padded_linear, vk::PipelineStageFlagBits::eTransfer, - vk::AccessFlagBits::eTransferWrite); - scheduler.Finish(); - *linear = read_bytes(output, linear_capacity); - DestroyBuffer(&output); - DestroyBuffer(&input); - }; - const auto gpu_tile = [&](const std::vector& linear, std::vector* tiled, - uint64_t tiled_capacity, uint64_t linear_capacity, - std::span infos) { - const uint64_t padded_tiled = align_dword(tiled_capacity); - const uint64_t padded_linear = align_dword(linear_capacity); - auto input = - CreateHostBuffer(name, padded_linear, AllFlags, to_dwords(linear, padded_linear)); - auto output = CreateHostBuffer( - name, padded_tiled, AllFlags, - std::vector(static_cast(padded_tiled / sizeof(u32)), 0xabababab)); - tile_manager.Tile(input.buffer, 0, padded_linear, output.buffer, 0, padded_tiled, - infos); - host_barrier(output.buffer, padded_tiled, vk::PipelineStageFlagBits::eComputeShader, - vk::AccessFlagBits::eShaderWrite); - scheduler.Finish(); - *tiled = read_bytes(output, tiled_capacity); - DestroyBuffer(&output); - DestroyBuffer(&input); - }; - - for (const bool d32: std::array {true, false}) { - constexpr uint32_t width = 5; - constexpr uint32_t height = 3; - constexpr uint32_t pitch = 8; - constexpr uint32_t layers = 2; - constexpr uint64_t d16_row = pitch * sizeof(uint16_t); - constexpr uint64_t d32_row = pitch * sizeof(uint32_t); - constexpr uint64_t d16_slice = d16_row * height; - constexpr uint64_t d32_slice = d32_row * height; - constexpr uint64_t d16_size = d16_slice * layers; - constexpr uint64_t d32_size = d32_slice * layers; - StreamBuffer conversion_upload(m_runtime_context, scheduler, MemoryUsage::Upload, - 1u << 20); - auto [prefix, prefix_offset] = conversion_upload.Map(4096, 256); - Require(name, "D16 conversion prefix", prefix != nullptr && prefix_offset == 0, - "failed to advance the D16 conversion upload ring"); - std::memset(prefix, 0x7b, 4096); - conversion_upload.Commit(); - auto [source_bytes, source_offset] = conversion_upload.Map(d16_size, 4); - Require(name, "D16 conversion source", source_bytes != nullptr && source_offset != 0, - "failed to reserve a nonzero D16 conversion source"); - std::memset(source_bytes, 0x3c, d16_size); - for (uint32_t layer = 0; layer < layers; layer++) { - for (uint32_t y = 0; y < height; y++) { - auto* row = - reinterpret_cast(source_bytes + layer * d16_slice + y * d16_row); - for (uint32_t x = 0; x < width; x++) { - row[x] = static_cast( - 0x1111u * (1u + x + y * width + layer * width * height)); - } - } - } - conversion_upload.Commit(); - auto promoted = - CreateHostBuffer(name, d32_size, AllFlags, - std::vector(static_cast(d32_size / sizeof(u32)), 0)); - auto round_trip = CreateHostBuffer( - name, d16_size, AllFlags, - std::vector(static_cast(d16_size / sizeof(u32)), 0xa5a5a5a5u)); - const TileManager::D16Layout promote_layout { - .width = width, - .height = height, - .layers = layers, - .source_row_stride = d16_row, - .target_row_stride = d32_row, - .source_slice_stride = d16_slice, - .target_slice_stride = d32_slice, - }; - tile_manager.ConvertD16({conversion_upload.Handle(), source_offset, - conversion_upload.Size() - source_offset}, - {promoted.buffer, 0, d32_size}, - TileManager::D16Direction::Promote, d32, promote_layout); - tile_manager.ConvertD16({promoted.buffer, 0, d32_size}, - {round_trip.buffer, 0, d16_size}, - TileManager::D16Direction::Demote, d32, - {.width = width, - .height = height, - .layers = layers, - .source_row_stride = d32_row, - .target_row_stride = d16_row, - .source_slice_stride = d32_slice, - .target_slice_stride = d16_slice}); - host_barrier(round_trip.buffer, d16_size, vk::PipelineStageFlagBits::eComputeShader, - vk::AccessFlagBits::eShaderWrite); - scheduler.Finish(); - const auto result = read_bytes(round_trip, d16_size); - bool conversion_matches = true; - for (uint32_t layer = 0; layer < layers; layer++) { - for (uint32_t y = 0; y < height; y++) { - for (uint32_t x = 0; x < pitch; x++) { - const auto offset = layer * d16_slice + y * d16_row + x * sizeof(uint16_t); - conversion_matches &= - x < width ? std::memcmp(result.data() + offset, source_bytes + offset, - sizeof(uint16_t)) == 0 - : result[offset] == 0xa5 && result[offset + 1] == 0xa5; - } - } - } - Require(name, d32 ? "D32 conversion round trip" : "D24 conversion round trip", - conversion_matches, - "D16 conversion lost rows, layers, active values, or padding"); - DestroyBuffer(&round_trip); - DestroyBuffer(&promoted); - } - - auto fill = [](std::vector* bytes, u32 salt) { - for (size_t i = 0; i < bytes->size(); i++) { - uint64_t value = i + static_cast(salt) * 0x9e3779b97f4a7c15ull; - value = (value ^ (value >> 30u)) * 0xbf58476d1ce4e5b9ull; - value = (value ^ (value >> 27u)) * 0x94d049bb133111ebull; - (*bytes)[i] = static_cast((value ^ (value >> 31u)) >> 56u); - } - }; - auto compare = [&](const char* stage, const std::vector& expected, - const std::vector& actual) { - if (expected != actual) { - const auto mismatch = static_cast( - std::mismatch(expected.begin(), expected.end(), actual.begin()).first - - expected.begin()); - std::ostringstream out; - out << "first mismatch at " << mismatch << " of " << expected.size(); - Fail(name, stage, out.str()); - } - }; - auto convert_reference = [&](bool to_tiled, std::vector* dst, - const std::vector& src, const GpuTileInfo& info) { - TileBlockLayout block {}; - Require(name, "reference layout", - TileGetBlockLayout(info.family, info.bytes_per_element, block), - "CPU reference rejected GPU tile info"); - const u32 tiled_width = info.tiled_width != 0 ? info.tiled_width : info.pitch; - const u32 tiled_height = info.tiled_height != 0 ? info.tiled_height : info.height; - const uint64_t columns = (tiled_width + block.block_width - 1u) / block.block_width; - const uint64_t rows = (tiled_height + block.block_height - 1u) / block.block_height; - const uint64_t slice = - info.linear_slice_stride != 0 - ? info.linear_slice_stride - : static_cast(info.pitch) * info.height * info.bytes_per_element; - for (u32 z = 0; z < info.depth; ++z) { - for (u32 y = 0; y < info.height; ++y) { - for (u32 x = 0; x < info.width; ++x) { - const u32 bx = info.tail ? 0 : x / block.block_width; - const u32 by = info.tail ? 0 : y / block.block_height; - const u32 bz = info.tail ? 0 : z / block.block_depth; - const u32 lx = info.tail ? x + info.tail_x : x % block.block_width; - const u32 ly = info.tail ? y + info.tail_y : y % block.block_height; - const u32 lz = z % block.block_depth; - u32 local = 0, block_xor = 0; - Require(name, "reference offset", - TileGetBlockOffset(block, lx, ly, lz, local) && - TileGetBlockXor(block, bx, by, info.surface_z + bz, block_xor), - "CPU reference address lookup failed"); - const uint64_t block_index = - static_cast(bz) * columns * rows + by * columns + bx; - const uint64_t tiled = info.tiled_offset + block_index * block.block_size + - (local ^ block_xor); - const uint64_t linear = - info.linear_offset + static_cast(z) * slice + - static_cast(y) * info.pitch * info.bytes_per_element + - static_cast(x) * info.bytes_per_element; - const uint64_t dst_offset = to_tiled ? tiled : linear; - const uint64_t src_offset = to_tiled ? linear : tiled; - Require(name, "reference range", - dst_offset + info.bytes_per_element <= dst->size() && - src_offset + info.bytes_per_element <= src.size(), - "CPU reference address escaped storage"); - std::memcpy(dst->data() + dst_offset, src.data() + src_offset, - info.bytes_per_element); - } - } - } - }; - struct FamilyCase { - TileBlockFamily family; - u32 max_bpe; - }; - constexpr FamilyCase families[] = { - {TileBlockFamily::Standard256B, 16}, {TileBlockFamily::Standard4KB, 16}, - {TileBlockFamily::Standard4KB3D, 16}, {TileBlockFamily::Standard64KB, 16}, - {TileBlockFamily::Standard64KB3D, 16}, {TileBlockFamily::Prt64KB, 16}, - {TileBlockFamily::Prt64KB3D, 16}, {TileBlockFamily::RenderTarget64KB, 16}, - {TileBlockFamily::Depth64KB, 8}, - }; - - { - TileBlockLayout standard {}, prt {}, standard_3d {}, prt_3d {}, color {}, depth {}; - u32 standard_offset = 0, prt_offset = 0, standard_3d_offset = 0, prt_3d_offset = 0, - color_z = 0, depth_z = 0; - bool fixed_addresses = - TileGetBlockLayout(TileBlockFamily::Standard64KB, 4, standard) && - TileGetBlockLayout(TileBlockFamily::Prt64KB, 4, prt) && - TileGetBlockLayout(TileBlockFamily::Standard64KB3D, 4, standard_3d) && - TileGetBlockLayout(TileBlockFamily::Prt64KB3D, 4, prt_3d) && - TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, color) && - TileGetBlockLayout(TileBlockFamily::Depth64KB, 4, depth) && - TileGetBlockOffset(standard, 64, 0, 0, standard_offset) && - TileGetBlockOffset(prt, 64, 0, 0, prt_offset) && - TileGetBlockOffset(standard_3d, 16, 0, 0, standard_3d_offset) && - TileGetBlockOffset(prt_3d, 16, 0, 0, prt_3d_offset) && - TileGetBlockXor(color, 0, 0, 1, color_z) && - TileGetBlockXor(depth, 0, 0, 15, depth_z); - Require(name, "fixed address vectors", - fixed_addresses && standard_offset == 0x8000 && prt_offset == 0x8100 && - standard_3d_offset == 0x8000 && prt_3d_offset == 0x8400 && - standard_3d.block_width == 32 && standard_3d.block_height == 32 && - standard_3d.block_depth == 16 && color_z == 0x800 && depth_z == 0xf00, - "a fixed block address changed"); - - constexpr auto format = Prospero::BufferFormat::k32Float; - constexpr u32 levels = 7; - TileSizeAlign total {}; - TileSizeOffset mip[levels] {}; - TilePaddedSize padded[levels] {}; - TileGetTextureSize(format, 65, 33, levels, - Prospero::TileMode::kStandard256B, &total, - mip, padded); - constexpr u32 offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300, 0x200, 0x100, 0}; - constexpr u32 sizes[levels] = {0x2d00, 0xf00, 0x600, 0x200, 0x100, 0x100, 0x100}; - bool layout_matches = total.size == 0x4700 && total.align == 0x100 && - padded[0].width == 72 && padded[0].height == 40; - for (u32 level = 0; level < levels; ++level) { - layout_matches &= - mip[level].offset == offsets[level] && mip[level].size == sizes[level]; - } - Require(name, "fixed mip vector", layout_matches, - "the reverse-packed small-block mip layout changed"); - } - - u32 case_index = 0; - auto check_round_trip = [&](const char* stage, uint64_t tiled_size, - std::span infos) { - uint64_t linear_size = 0; - for (const auto& info: infos) { - linear_size = std::max(linear_size, info.linear_offset + info.linear_size); - } - std::vector tiled(tiled_size); - std::vector cpu(linear_size, 0); - std::vector gpu(linear_size, 0xab); - fill(&tiled, ++case_index); - for (const auto& info: infos) { - convert_reference(false, &cpu, tiled, info); - } - gpu_detile(tiled, &gpu, tiled_size, linear_size, infos); - compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu); - - std::vector linear(linear_size); - std::vector cpu_tiled(tiled_size, 0xab); - std::vector gpu_tiled(tiled_size, 0xab); - fill(&linear, 0x280u + case_index); - for (const auto& info: infos) { - convert_reference(true, &cpu_tiled, linear, info); - } - gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, infos); - compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled, gpu_tiled); - }; - for (const auto family: families) { - for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) { - TileBlockLayout block {}; - Require(name, "block layout", TileGetBlockLayout(family.family, bpe, block), - "admitted family/BPE has no block layout"); - const bool volume = block.block_depth > 1; - const u32 width = block.block_width * 3u + std::min(block.block_width, 3u); - const u32 height = block.block_height * 3u + std::min(block.block_height, 3u); - const u32 depth = volume ? block.block_depth + 1u : 1u; - const u32 pitch = block.block_width * 4u; - const uint64_t block_columns = (pitch + block.block_width - 1u) / block.block_width; - const uint64_t block_rows = (height + block.block_height - 1u) / block.block_height; - const uint64_t block_slices = (depth + block.block_depth - 1u) / block.block_depth; - const uint64_t storage_size = - block_columns * block_rows * block_slices * block.block_size; - const uint64_t slice_stride = static_cast(pitch) * height * bpe; - - std::vector tiled(storage_size); - std::vector cpu(storage_size, 0); - std::vector gpu(storage_size, 0xab); - fill(&tiled, ++case_index); - - GpuTileInfo info {}; - info.family = block.family; - info.bytes_per_element = block.bytes_per_element; - info.linear_size = storage_size; - info.tiled_size = storage_size; - info.linear_slice_stride = volume ? slice_stride : 0; - info.width = width; - info.height = height; - info.depth = depth; - info.pitch = pitch; - info.surface_z = family.family == TileBlockFamily::RenderTarget64KB || - family.family == TileBlockFamily::Depth64KB - ? 3 - : 0; - convert_reference(false, &cpu, tiled, info); - gpu_detile(tiled, &gpu, storage_size, storage_size, - std::span(&info, 1)); - const auto family_label = [&](const char* operation) { - std::ostringstream out; - out << operation << " family=" << static_cast(family.family) - << " bpe=" << bpe; - return out.str(); - }; - compare(family_label("detile bytes").c_str(), cpu, gpu); - - { - std::vector linear(storage_size); - std::vector cpu_tiled(storage_size, 0xab); - std::vector gpu_tiled(storage_size, 0xab); - fill(&linear, 0x80u + case_index); - convert_reference(true, &cpu_tiled, linear, info); - gpu_tile(linear, &gpu_tiled, storage_size, storage_size, - std::span(&info, 1)); - compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled); - } - } - } - - // Exercise the real texture-layout/info-building seam for every format - // admitted by each standard tile mode. Formats sharing a byte/block width - // intentionally share a shader, but this loop still validates their - // texel-to-element conversion (notably every BCn format). - struct StandardMode { - Prospero::TileMode tile; - TileBlockFamily family; - }; - constexpr StandardMode standard_modes[] = { - {Prospero::TileMode::kStandard256B, - TileBlockFamily::Standard256B}, - {Prospero::TileMode::kStandard4KB, - TileBlockFamily::Standard4KB}, - {Prospero::TileMode::kStandard64KB, - TileBlockFamily::Standard64KB}, - {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB}, - }; - for (const auto format: {Prospero::BufferFormat::k8Srgb, - Prospero::BufferFormat::k8_8Srgb, - Prospero::BufferFormat::k9_9_9_5Float}) { - for (const auto tile: {Prospero::TileMode::kDepth, - Prospero::TileMode::kRenderTarget}) { - TileTextureBlockLayout texture {}; - Require(name, "RT format policy", - !TileGetTextureBlockLayout(format, tile, false, texture), - "non-render-target format admitted by an RT/depth tile family"); - } - } - { - TileSurfaceLayout invalid {}; - const TileSurfaceDescription description { - Prospero::BufferFormat::k32Float, - Prospero::TileMode::kStandard256B, - TileSurfaceDimension::Dim3D, - 16, - 16, - 16, - 1, - 1, - }; - Require(name, "3D tile policy", !TileGetTiledTextureLayout(description, invalid), - "Standard256B volume silently fell back to a 2D array layout"); - } - { - constexpr auto format = - Prospero::BufferFormat::kFmask8_S4_F4; - constexpr auto tile = Prospero::TileMode::kStandard64KB; - TileSizeAlign total {}; - TileGetTextureSize(format, 128, 128, 1, tile, &total, nullptr, nullptr); - const auto layout = TextureCalcUploadLayout(format, 128, 128, 1, 1, tile, total.size, - false, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - Require(name, "FMASK policy", - !TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos), - "FMASK entered the texel tiler through a non-depth family"); - } - u32 format_cases = 0; - for (u32 raw_format = 1; - raw_format <= static_cast(Prospero::BufferFormat::kBc7Srgb); - ++raw_format) { - const auto format = static_cast(raw_format); - // FMASK is synthesized as identity metadata by TextureUploadFmask; it is - // deliberately not a texel surface and must never enter the GPU tiler. - if (Prospero::IsFmaskTextureFormat(format)) { - continue; - } - for (const auto& mode: standard_modes) { - TileTextureBlockLayout texture {}; - if (!TileGetTextureBlockLayout(format, mode.tile, false, texture)) { - continue; - } - const u32 bpe = texture.block.bytes_per_element; - Require(name, "format block", texture.block.family == mode.family, - "CPU-supported format selected the wrong block family"); - - constexpr u32 width = 67; - constexpr u32 height = 51; - const u32 pitch = TileGetTexturePitch(format, width, mode.tile); - TileSizeAlign total {}; - TileGetTextureSize(format, width, height, 1, mode.tile, &total, nullptr, - nullptr); - Require(name, "format size", total.size != 0, - "supported format has an empty layout"); - - const auto layout = TextureCalcUploadLayout( - format, width, height, 1, 1, mode.tile, total.size, false, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - if (!TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos)) { - std::ostringstream out; - out << "format=" << raw_format - << " tile=" << static_cast(mode.tile) - << " size=" << total.size << " pitch=" << pitch; - Fail(name, "format infos", out.str()); - } - Require(name, "format family", - infos.size() == 1 && infos[0].family == mode.family && - infos[0].bytes_per_element == bpe, - "texture info selected the wrong shader family"); - - std::vector tiled(total.size); - std::vector cpu(total.size, 0); - std::vector gpu(total.size, 0xab); - fill(&tiled, ++case_index); - convert_reference(false, &cpu, tiled, infos[0]); - gpu_detile(tiled, &gpu, total.size, total.size, infos); - compare("format bytes", cpu, gpu); - ++format_cases; - } - } - Require(name, "format coverage", format_cases != 0, - "no CPU-supported standard formats were tested"); - - { - constexpr auto format = Prospero::BufferFormat::k32_32_32Float; - TileTextureElementLayout element {}; - Require(name, "RGB32 texture policy", !TileGetTextureElementLayout(format, element), - "12-byte texel layouts must be rejected"); - } - - { - constexpr auto format = Prospero::BufferFormat::kBc1UNorm; - constexpr auto tile = Prospero::TileMode::kStandard64KB; - constexpr u32 width = 256, height = 256, levels = 9; - TileSizeAlign total {}; - TileGetTextureSize(format, width, height, levels, tile, &total, nullptr, - nullptr); - const auto layout = TextureCalcUploadLayout(format, width, height, levels, 1, tile, - total.size, false, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); - uint64_t linear_size = 0; - for (const auto& info: infos) { - linear_size = std::max(linear_size, info.linear_offset + info.linear_size); - } - Require(name, "BC1 mip-tail capacities", - built && total.size == 0x10000 && layout.surface.first_tail_level == 0 && - linear_size == 0x15560 && linear_size > total.size, - "BC1 mip tail conflated tiled and linear capacities"); - check_round_trip("BC1 mip tail", total.size, infos); - } - - { - constexpr auto format = Prospero::BufferFormat::k32Float; - constexpr auto tile = Prospero::TileMode::kRenderTarget; - constexpr u32 width = 129, height = 65, layers = 3; - TileSizeAlign total {}; - TileGetTextureTotalSize(format, width, height, layers, 1, tile, false, total); - const auto layout = TextureCalcUploadLayout(format, width, height, 1, layers, tile, - total.size, true, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos); - Require(name, "array infos", - built && infos.size() == layers && infos[0].surface_z == 0 && - infos[1].surface_z == 1 && infos[2].surface_z == 2, - "array slices lost their absolute surface Z"); - check_round_trip("array", total.size, infos); - } - - for (const auto& mode: standard_modes) { - constexpr auto format = Prospero::BufferFormat::k32Float; - constexpr u32 levels = 2; - TileBlockLayout block {}; - Require(name, "odd mip block", TileGetBlockLayout(mode.family, 4, block), - "odd multi-mip format has no block layout"); - const u32 width = block.block_width * 2u + 1u; - const u32 height = block.block_height * 2u + 1u; - TileSizeAlign total {}; - TileGetTextureSize(format, width, height, levels, mode.tile, &total, nullptr, - nullptr); - const auto layout = TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile, - total.size, false, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - Require( - name, "odd mip infos", - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos) && - infos.size() == 2 && infos[1].tiled_width >= infos[1].pitch && - infos[1].tiled_height >= infos[1].height && - (mode.family == TileBlockFamily::Standard256B || - infos[1].tiled_height > infos[1].height), - "odd multi-mip physical stride collapsed to the active linear " - "extent"); - check_round_trip("odd multi-mip", total.size, infos); - } - - struct TailMode { - Prospero::TileMode tile; - TileBlockFamily family; - }; - constexpr TailMode tail_modes[] = { - {Prospero::TileMode::kStandard4KB, - TileBlockFamily::Standard4KB}, - {Prospero::TileMode::kStandard64KB, - TileBlockFamily::Standard64KB}, - {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB}, - {Prospero::TileMode::kRenderTarget, - TileBlockFamily::RenderTarget64KB}, - {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB}, - }; - for (const auto& mode: tail_modes) { - constexpr auto format = Prospero::BufferFormat::k32Float; - constexpr u32 levels = 7; - TileBlockLayout block {}; - Require(name, "2D tail block", TileGetBlockLayout(mode.family, 4, block), - "2D tail mode has no block layout"); - const u32 width = block.block_width * 2u; - const u32 height = block.block_height * 2u; - TileSizeAlign total {}; - TileGetTextureSize(format, width, height, levels, mode.tile, &total, nullptr, - nullptr); - const auto layout = TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile, - total.size, true, false, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - Require(name, "2D mip tail seam", - layout.surface.first_tail_level == 2 && - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos) && - infos.size() == levels && !infos[0].tail && !infos[1].tail && - std::all_of(infos.begin() + 2, infos.end(), - [](const auto& info) { return info.tail; }), - "2D mip chain lost its linear/tiled tail boundary"); - check_round_trip("2D mip tail seam", total.size, infos); - } - - { - constexpr auto format = Prospero::BufferFormat::k32Float; - constexpr auto tile = Prospero::TileMode::kStandard4KB; - constexpr u32 width = 65; - constexpr u32 height = 129; - constexpr u32 depth = 17; - constexpr u32 levels = 2; - TileSizeAlign total {}; - TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, total); - const auto layout = TextureCalcUploadLayout(format, width, height, levels, depth, tile, - total.size, false, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); - Require(name, "3D mip infos", - regions.size() == depth + (depth >> 1u) && built && infos.size() == 4 && - infos[0].depth == 8 && infos[1].depth == 8 && infos[2].depth == 1 && - infos[3].depth == 8 && infos[0].tiled_offset == 0x19000 && - infos[1].tiled_offset == 0x83000 && infos[2].tiled_offset == 0xed000 && - infos[3].tiled_offset == 0 && infos[3].pitch == 32 && - infos[3].height == 64 && infos[3].tiled_width == 40 && - infos[3].tiled_height == 80, - "Standard4KB3D mip depth, packing, or physical stride changed"); - check_round_trip("3D mip", total.size, infos); - } - - { - constexpr auto format = Prospero::BufferFormat::kBc1UNorm; - constexpr auto tile = Prospero::TileMode::kStandard4KB; - constexpr u32 width = 65; - constexpr u32 height = 129; - constexpr u32 depth = 17; - constexpr u32 levels = 2; - TileSizeAlign total {}; - TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, total); - const auto layout = TextureCalcUploadLayout(format, width, height, levels, depth, tile, - total.size, false, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - Require(name, "3D BC mip infos", - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos) && - infos.size() == 4 && infos[3].pitch == 8 && infos[3].height == 16 && - infos[3].tiled_width == 16 && infos[3].tiled_height == 24, - "block-compressed 3D mip lost its physical row or slice stride"); - check_round_trip("3D BC mip", total.size, infos); - } - - { - constexpr auto format = Prospero::BufferFormat::kBc3UNorm; - constexpr auto tile = Prospero::TileMode::kLinear; - constexpr u32 width = 8, height = 8, levels = 4; - TileSizeAlign total {}; - TileGetTextureSize(format, width, height, levels, tile, &total, nullptr, - nullptr); - const auto layout = TextureCalcUploadLayout(format, width, height, levels, 1, tile, - total.size, false, false, name); - const auto regions = TextureBuildImageCopies(layout); - Require(name, "linear BC native regions", - regions.size() == levels && - std::all_of(regions.begin(), regions.end(), - [](const auto& region) { - return region.bufferRowLength >= region.imageExtent.width && - region.bufferImageHeight >= - region.imageExtent.height && - region.bufferRowLength % 4 == 0 && - region.bufferImageHeight % 4 == 0; - }), - "linear BC mip tails emitted invalid Vulkan copy strides"); - } - - struct VolumeModeCase { - Prospero::TileMode tile; - TileBlockFamily family; - Prospero::BufferFormat format; - }; - constexpr VolumeModeCase volume_modes[] = { - {Prospero::TileMode::kStandard64KB, - TileBlockFamily::Standard64KB3D, - Prospero::BufferFormat::k32Float}, - {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D, - Prospero::BufferFormat::k32Float}, - {Prospero::TileMode::kRenderTarget, - TileBlockFamily::RenderTarget64KB, - Prospero::BufferFormat::k32Float}, - {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB, - Prospero::BufferFormat::k32Float}, - {Prospero::TileMode::kStandard64KB, - TileBlockFamily::Standard64KB3D, - Prospero::BufferFormat::kBc1UNorm}, - {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D, - Prospero::BufferFormat::kBc3UNorm}, - }; - for (const auto test: volume_modes) { - constexpr u32 width = 65, height = 33, depth = 37, levels = 6; - TileSizeAlign total {}; - TileGetTextureTotalSize(test.format, width, height, depth, levels, test.tile, true, - total); - const auto layout = - TextureCalcUploadLayout(test.format, width, height, levels, depth, test.tile, - total.size, true, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - const bool built = TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); - const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB || - test.family == TileBlockFamily::Depth64KB; - Require( - name, "volume family infos", - built && !infos.empty() && - std::all_of(infos.begin(), infos.end(), - [&](const auto& info) { return info.family == test.family; }) && - (!uses_z || std::any_of(infos.begin(), infos.end(), - [](const auto& info) { return info.surface_z != 0; })), - "volume mode selected the wrong family or lost its surface Z"); - check_round_trip("volume family", total.size, infos); - } - - struct VolumeTailCase { - Prospero::BufferFormat format; - u32 bytes; - }; - constexpr VolumeTailCase volume_tails[] = { - {Prospero::BufferFormat::k8UNorm, 1}, - {Prospero::BufferFormat::k16UNorm, 2}, - {Prospero::BufferFormat::k32Float, 4}, - {Prospero::BufferFormat::k16_16_16_16Float, 8}, - {Prospero::BufferFormat::k32_32_32_32Float, 16}, - {Prospero::BufferFormat::kBc1UNorm, 8}, - {Prospero::BufferFormat::kBc3UNorm, 16}, - }; - constexpr u32 volume_tail_xy[5][5][2] = { - {{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}}, {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, - {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, {{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}}, - {{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}}, - }; - for (const auto& tail: volume_tails) { - constexpr auto tile = Prospero::TileMode::kStandard4KB; - constexpr u32 levels = 5; - TileBlockLayout block {}; - Require(name, "3D tail block", - TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block), - "Standard4KB3D tail format has no block layout"); - const bool compressed = Prospero::BlockCompressedBytesPerBlock(tail.format) != 0; - const u32 width = block.block_width * (compressed ? 4u : 1u); - const u32 height = block.block_height / 2u * (compressed ? 4u : 1u); - const u32 depth = block.block_depth + 1u; - TileSizeAlign total {}; - TileGetTextureTotalSize(tail.format, width, height, depth, levels, tile, true, - total); - const auto layout = TextureCalcUploadLayout(tail.format, width, height, levels, depth, - tile, total.size, false, true, name); - const auto regions = TextureBuildImageCopies(layout); - std::vector infos; - bool valid = - total.size == 8192 && - regions.size() == depth + std::max(depth >> 1u, 1u) + std::max(depth >> 2u, 1u) + - std::max(depth >> 3u, 1u) + std::max(depth >> 4u, 1u) && - TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos) && - infos.size() == 6; - const u32 table = std::countr_zero(tail.bytes); - for (u32 level = 0; level < levels && valid; level++) { - const auto& info = infos[level == 0 ? 0 : level + 1]; - valid &= info.tail && info.tail_x == volume_tail_xy[table][level][0] && - info.tail_y == volume_tail_xy[table][level][1] && info.tiled_offset == 0; - } - if (valid) { - valid = infos[1].tail && infos[1].tail_x == volume_tail_xy[table][0][0] && - infos[1].tail_y == volume_tail_xy[table][0][1] && - infos[1].tiled_offset == 4096; - } - Require(name, "3D mip tail infos", valid, - "Standard4KB3D mip tail coordinates or block-slice packing changed"); - check_round_trip("3D mip tail", total.size, infos); - } - - for (const auto family: - {TileBlockFamily::Standard4KB, TileBlockFamily::Standard64KB, TileBlockFamily::Prt64KB, - TileBlockFamily::RenderTarget64KB, TileBlockFamily::Depth64KB}) { - for (u32 bpe = 1; bpe <= 16; bpe <<= 1u) { - if (family == TileBlockFamily::Depth64KB && bpe == 16) continue; - TileBlockLayout block {}; - Require(name, "tail layout", TileGetBlockLayout(family, bpe, block), - "tail family/BPE has no block layout"); - const u32 width = std::max(block.block_width / 4u, 1u); - const u32 height = std::max(block.block_height / 4u, 1u); - const u32 x = block.block_width / 2u; - const u32 y = block.block_height / 2u; - const u32 pitch = width; - const uint64_t linear_size = static_cast(pitch) * height * bpe; - std::vector tiled(block.block_size); - std::vector cpu(linear_size, 0xcd); - std::vector gpu(linear_size, 0xab); - fill(&tiled, ++case_index); - GpuTileInfo info {}; - info.family = block.family; - info.bytes_per_element = block.bytes_per_element; - info.linear_size = linear_size; - info.tiled_size = block.block_size; - info.width = width; - info.height = height; - info.pitch = pitch; - info.tail = true; - info.tail_x = x; - info.tail_y = y; - info.surface_z = family == TileBlockFamily::RenderTarget64KB || - family == TileBlockFamily::Depth64KB - ? 2 - : 0; - convert_reference(false, &cpu, tiled, info); - gpu_detile(tiled, &gpu, block.block_size, linear_size, - std::span(&info, 1)); - compare("tail bytes", cpu, gpu); - - std::vector linear(linear_size); - std::vector cpu_tiled(block.block_size, 0xab); - std::vector gpu_tiled(block.block_size, 0xab); - fill(&linear, 0x400u + case_index); - convert_reference(true, &cpu_tiled, linear, info); - gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, - std::span(&info, 1)); - compare("tail tile bytes", cpu_tiled, gpu_tiled); - } - } - - TileBlockLayout small_block {}; - Require(name, "small layout", - TileGetBlockLayout(TileBlockFamily::Standard256B, 4, small_block), - "small fixture layout is unavailable"); - std::vector small_input(small_block.block_size); - std::vector small_expected(small_block.block_size, 0); - std::vector small_output(small_block.block_size, 0xab); - fill(&small_input, 0xee); - GpuTileInfo small_info {}; - small_info.family = small_block.family; - small_info.bytes_per_element = small_block.bytes_per_element; - small_info.linear_size = small_output.size(); - small_info.tiled_size = small_input.size(); - small_info.width = 1; - small_info.height = 1; - small_info.pitch = small_block.block_width; - convert_reference(false, &small_expected, small_input, small_info); - gpu_detile(small_input, &small_output, small_input.size(), small_output.size(), - std::span(&small_info, 1)); - compare("small detile", small_expected, small_output); - - std::fill(small_output.begin(), small_output.end(), 0xab); - gpu_detile(small_input, &small_output, small_input.size(), small_output.size(), - std::span(&small_info, 1)); - compare("scheduler-owned reuse", small_expected, small_output); - - constexpr auto volume_format = Prospero::BufferFormat::k32UInt; - constexpr auto volume_tile = Prospero::TileMode::kLinear; - constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5; - constexpr u32 volume_levels = 3; - const u32 volume_pitch = - TileGetTexturePitch(volume_format, volume_width, volume_tile); - TileSizeAlign volume_size {}; - TileGetTextureTotalSize(volume_format, volume_width, volume_height, volume_depth, - volume_levels, volume_tile, true, volume_size); - const auto volume_layout = TextureCalcUploadLayout( - volume_format, volume_width, volume_height, volume_levels, volume_depth, volume_tile, - volume_size.size, true, true, name); - const auto volume_copies = TextureBuildImageCopies(volume_layout); - std::vector volume_source(volume_size.size / sizeof(u32), 0); - std::vector> volume_probes; - for (const auto& copy: volume_copies) { - for (u32 y = 0; y < copy.imageExtent.height; y++) { - for (u32 x = 0; x < copy.imageExtent.width; x++) { - const auto index = - (copy.bufferOffset + - (static_cast(y) * copy.bufferRowLength + x) * sizeof(u32)) / - sizeof(u32); - const u32 value = 0xa0000000u | (copy.imageSubresource.mipLevel << 24u) | - (static_cast(copy.imageOffset.z) << 16u) | (y << 8u) | x; - Require(name, "linear volume native bounds", index < volume_source.size(), - "linear volume copy escaped its padded guest layout"); - volume_source[index] = value; - volume_probes.emplace_back(static_cast(index), value); - } - } - } - auto volume_upload = CreateHostBuffer(name, volume_size.size, - vk::BufferUsageFlagBits::eTransferSrc, volume_source); - auto volume_download = - CreateHostBuffer(name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst, - std::vector(volume_source.size(), 0)); - ImageInfo volume_info {}; - volume_info.data = {0x10000, volume_size.size}; - volume_info.pixel_format = vk::Format::eR32Uint; - volume_info.guest_format = volume_format; - volume_info.type = Prospero::ImageType::kColor3D; - volume_info.extent = {volume_width, volume_height, volume_depth}; - volume_info.resources = {volume_levels, 1}; - volume_info.pitch = volume_pitch; - volume_info.bytes_per_block = sizeof(u32); - volume_info.tile_mode = volume_tile; - for (u32 level = 0; level < volume_levels; level++) { - volume_info.mip_layout[level] = {volume_layout.mips[level].offset, - volume_layout.mips[level].size, - volume_layout.mips[level].row_length, - volume_layout.mips[level].image_height}; - } - Libs::Graphics::Image volume_image(m_runtime_context, scheduler, volume_info); - volume_image.Upload(volume_copies, volume_upload.buffer, 0, volume_size.size); - volume_image.Download(volume_copies, volume_download.buffer, 0, volume_size.size); - scheduler.Finish(); - const auto volume_observed = ReadBuffer(name, volume_download, volume_source.size()); - Require(name, "linear volume native content", - std::ranges::all_of(volume_probes, - [&](const auto& probe) { - return volume_observed[probe.first] == probe.second; - }), - "native upload/download did not preserve padded 3D mip rows"); - DestroyBuffer(&volume_download); - DestroyBuffer(&volume_upload); - std::printf("[gpu] %-32s ok (%u cases, %u format/mode pairs)\n", name, case_index, - format_cases); - } + "backing and 2D " + "attachment slice"); + + RenderExecutorTestAccess::ResetBindings(executor); + registers.SetColorView( + 0, {.base_array_slice_index = 7, .last_array_slice_index = 7}); + RenderColorInfo sliced_color{}; + RenderExecutorTestAccess::ResolveRenderColorTarget( + executor, 2, scheduler.Current(), sliced_color, 0); + RenderDepthInfo no_depth{}; + const auto sliced_rendering = + RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &sliced_color, 1, no_depth); + Require( + name, "3D slice transition", + sliced_color.image_id == color.image_id && + sliced_color.image_view != nullptr && + sliced_color.desc.view_info.base_layer == 7 && + sliced_rendering.num_color_attachments == 1 && + sliced_rendering.num_layers == 1, + "a nonzero 3D attachment slice was treated as a Vulkan array layer"); + + auto storage_desc = color.desc; + storage_desc.type = BindingType::Storage; + storage_desc.info.guest_format = Prospero::BufferFormat::k10_10_10_2UNorm; + storage_desc.view_info.type = vk::ImageViewType::e3D; + storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + const auto storage_id = texture_cache.FindImage(storage_desc); + const auto storage_view = + texture_cache.FindTexture(storage_id, storage_desc); + const auto &shared_image = texture_cache.GetImage(storage_id); + Require(name, "storage alias reuse", + storage_id == color.image_id && storage_view != nullptr && + shared_image.IsGpuModified() && + shared_image.usage.render_target, + "the matching 3D storage binding did not reuse the live " + "render-target image"); + + Require( + name, "volume readback queue", + TextureCacheTestAccess::TryDownload(texture_cache, storage_id), + "the 3D render target could not be queued for guest-layout readback"); + auto mirror = resources.GetBufferCache().ObtainBuffer( + scheduler.Current(), base, allocation_size, false, true, true); + Require(name, "volume mirror", + mirror.buffer != nullptr && mirror.owner != nullptr, + "the 3D render-target readback has no BufferCache owner"); + scheduler.Current().RetainResourceUntilFence(mirror.owner); + auto slice_probe = + CreateHostBuffer(name, 4, vk::BufferUsageFlagBits::eTransferDst, {0}); + const vk::BufferCopy copy{mirror.offset + 31 * slice_size, 0, 4}; + scheduler.Current().Handle().copyBuffer(mirror.buffer, slice_probe.buffer, + 1, ©); + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = slice_probe.buffer; + barrier.size = slice_probe.size; + scheduler.Current().Handle().pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, + nullptr); + scheduler.Finish(); + scheduler.DrainPriorityOperations(); + Require(name, "volume Z transfer", + ReadBuffer(name, slice_probe, 1) == std::vector{0x5a5a5a5a}, + "render-target upload/readback lost the final Z slice"); + DestroyBuffer(&slice_probe); + + RenderExecutorTestAccess::ResetBindings(executor); + resources.UnmapMemory(base, allocation_size); + scheduler.Finish(); + } + + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "color-volume direct mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "color-volume direct-memory allocation release failed"); + std::printf("[gpu] %-32s ok\n", name); + } + + void CheckRenderExecutorStencilBindingDiscovery() { + constexpr const char *name = "RenderExecutorStencilBindingDiscovery"; + constexpr uintptr_t base = 0x0000000203600000ull; + constexpr uint64_t allocation_size = 0x180000; + constexpr uint64_t allocation_alignment = 0x10000; + constexpr uint64_t depth_address = base + 0x40000; + constexpr uint64_t stencil_address = base + 0x70000; + EnsureRuntimeContext(); + + int64_t direct_offset = -1; + Require(name, "direct allocation", + Libs::LibKernel::Memory::KernelAllocateDirectMemory( + 0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(), + allocation_size, allocation_alignment, 0, &direct_offset) == 0, + "descriptor discovery direct-memory allocation failed"); + void *mapped = reinterpret_cast(base); + Require(name, "direct mapping", + Libs::LibKernel::Memory::KernelMapDirectMemory( + &mapped, allocation_size, 0x3, 0x10, direct_offset, + allocation_alignment) == 0 && + mapped == reinterpret_cast(base), + "descriptor discovery fixed mapping failed"); + std::memset(mapped, 0, allocation_size); + + { + RenderContext context(m_runtime_context); + auto &scheduler = context.GetCommandScheduler(); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + auto &resources = context.GetGpuResources(); + auto &texture_cache = resources.GetTextureCache(); + auto &executor = context.GetRenderExecutor(); + resources.MapMemory(base, allocation_size); + + constexpr auto stencil_format = Prospero::BufferFormat::k8UInt; + constexpr auto linear = Prospero::TileMode::kLinear; + TileSizeAlign stencil_layout{}; + TileGetTextureTotalSize(stencil_format, 1, 1, 1, 1, linear, false, + stencil_layout); + Require(name, "stencil footprint", + stencil_layout.size != 0 && stencil_layout.align != 0 && + (stencil_address & (stencil_layout.align - 1u)) == 0, + "linear stencil descriptor produced an invalid footprint"); + + ShaderRecompiler::IR::Program null_program{}; + null_program.stage = ShaderType::Compute; + null_program.values = + std::make_shared(); + null_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource null_resource{}; + null_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + null_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + null_resource.read = true; + null_program.info.images.push_back(null_resource); + auto null_volume = null_resource; + null_volume.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; + null_program.info.images.push_back(null_volume); + auto null_storage = null_resource; + null_storage.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + null_storage.read = false; + null_storage.written = true; + null_program.info.images.push_back(null_storage); + ShaderRecompiler::IR::ResourceSnapshot null_snapshot{}; + ShaderRecompiler::IR::DescriptorValue null_descriptor{}; + null_descriptor.dword_count = 8; + null_snapshot.images.assign(3, null_descriptor); + std::string null_error; + Require(name, "null specialization", + ShaderRecompiler::IR::SpecializeResources( + null_program, null_snapshot, &null_error), + null_error.c_str()); + ShaderStageRuntime null_runtime{ + std::make_shared( + std::move(null_program)), + std::make_shared( + std::move(null_snapshot))}; + auto null_bindings = executor.PrepareBindings( + scheduler.Current(), null_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), null_bindings); + Require(name, "null descriptor count", + null_bindings.resources.images.size() == 3, + "null descriptor preparation lost an image binding"); + const auto null_image_id = null_bindings.resources.images[0].image_id; + const auto &null_image = texture_cache.GetImage(null_image_id); + Require(name, "one null image per format", + null_bindings.resources.images[1].image_id == null_image_id && + null_bindings.resources.images[2].image_id == null_image_id && + TextureCacheTestAccess::NullImageCount(texture_cache) == 1, + "null view dimension or usage created another image allocation"); + Require(name, "null descriptors share final cache acquisition", + null_image.info.data.Empty() && !null_image.binding.is_bound && + !null_image.binding.is_target && + !null_image.binding.needs_rebind && + null_image.usage.texture && null_image.usage.storage && + null_image.IsGpuModified() && + !TextureCacheTestAccess::PendingDownload(texture_cache, + null_image_id), + "the shared null image did not preserve texture/storage " + "acquisition without guest readback or RenderExecutor ownership"); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + null_bindings); + Require(name, "null descriptor general layouts", + std::ranges::all_of( + null_bindings.resources.images, + [](const auto &binding) { + const auto info = + DescriptorCacheTestAccess::MakeImageInfo(binding); + return binding.layout == vk::ImageLayout::eGeneral && + info.imageView == binding.image_view && + info.imageLayout == binding.layout; + }) && + null_image.backing.state.layout == vk::ImageLayout::eGeneral, + "shared sampled/storage null descriptors did not retain " + "the general layout through descriptor generation"); + + constexpr uint64_t target_mip_size = 0x10000; + const auto make_target_desc = [&](uint64_t address, uint64_t size, + vk::Extent3D extent) { + ImageDesc desc{}; + desc.type = BindingType::RenderTarget; + desc.info.data = {address, size}; + desc.info.pixel_format = vk::Format::eR32Uint; + desc.info.guest_format = Prospero::BufferFormat::k32UInt; + desc.info.type = Prospero::ImageType::kColor2D; + desc.info.extent = extent; + desc.info.resources = {1, 1}; + desc.info.pitch = extent.width; + desc.info.bytes_per_block = 4; + desc.info.samples = 1; + desc.info.tile_mode = linear; + desc.info.mip_layout[0] = {0, size, extent.width, extent.height}; + desc.view_info.format = desc.info.pixel_format; + desc.view_info.type = vk::ImageViewType::e2D; + desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + desc.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + return desc; + }; + + TextureCacheTestAccess::SetLinearReadback(texture_cache, true); + ShaderTextureResource storage{}; + const uint64_t storage_address = base + 0x60000; + constexpr uint32_t storage_native_value = 0x13579bdfu; + constexpr uint32_t storage_stale_value = 0x2468ace0u; + std::memcpy(reinterpret_cast(mapped) + 0x60000, + &storage_native_value, sizeof(storage_native_value)); + const uint64_t encoded_storage_address = storage_address >> 8u; + storage.fields[0] = static_cast(encoded_storage_address); + storage.fields[1] = + static_cast(encoded_storage_address >> 32u) | + (static_cast(stencil_format) << 20u); + storage.fields[3] = + DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + storage.fields[5] = 0x00700000u; + ShaderRecompiler::IR::Program storage_program{}; + storage_program.stage = ShaderType::Vertex; + storage_program.values = + std::make_shared(); + storage_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource storage_resource{}; + storage_resource.kind = + ShaderRecompiler::IR::ResourceKind::StorageImageUint; + storage_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + storage_resource.written = true; + storage_program.info.images.push_back(storage_resource); + ShaderRecompiler::IR::DescriptorValue storage_descriptor{}; + std::copy(std::begin(storage.fields), std::end(storage.fields), + storage_descriptor.dwords.begin()); + storage_descriptor.dword_count = 8; + auto srgb_storage = storage; + constexpr auto srgb_format = + static_cast(Prospero::BufferFormat::k8_8_8_8Srgb); + constexpr uint64_t srgb_storage_address = base + 0xc0000; + const auto encoded_srgb_storage_address = srgb_storage_address >> 8u; + srgb_storage.fields[0] = + static_cast(encoded_srgb_storage_address); + srgb_storage.fields[1] = + static_cast(encoded_srgb_storage_address >> 32u) | + (srgb_format << 20u); + ShaderRecompiler::IR::DescriptorValue srgb_storage_descriptor{}; + std::copy(std::begin(srgb_storage.fields), std::end(srgb_storage.fields), + srgb_storage_descriptor.dwords.begin()); + srgb_storage_descriptor.dword_count = 8; + auto srgb_storage_resource = storage_resource; + srgb_storage_resource.kind = + ShaderRecompiler::IR::ResourceKind::StorageImage; + const auto srgb_storage_binding = + RenderExecutorTestAccess::ResolveTexture( + executor, srgb_storage_resource, srgb_storage_descriptor); + const auto srgb_storage_view = texture_cache.FindTexture( + srgb_storage_binding.image_id, srgb_storage_binding.desc); + Require(name, "sRGB storage view", + srgb_storage_view != nullptr && + srgb_storage_binding.desc.info.pixel_format == + vk::Format::eR8G8B8A8Srgb && + srgb_storage_binding.desc.view_info.format == + vk::Format::eR8G8B8A8Unorm && + texture_cache.GetImage(srgb_storage_binding.image_id) + .backing.format == vk::Format::eR8G8B8A8Srgb, + "storage descriptor did not preserve its sRGB backing and " + "select an UNORM Vulkan view"); + + ShaderTextureResource sint_storage{{0x01514b00u, 0xc1500000u, 0x000bc00bu, + 0x91b00204u, 0x00000000u, 0x00700000u, + 0x102b0000u, 0x0001514au}}; + Require(name, "PPSA06888 R32 SINT descriptor", + sint_storage.Base40() == 0x1514b0000ull && + sint_storage.Width5() + 1u == 48 && + sint_storage.Height5() + 1u == 48 && + sint_storage.Depth() + 1u == 1 && + sint_storage.Format() == Prospero::BufferFormat::k32SInt && + sint_storage.Type() == Prospero::ImageType::kColor2D && + sint_storage.TileMode() == + Prospero::TileMode::kRenderTarget && + sint_storage.DstSelXYZW() == DstSel(4, 0, 0, 1), + "captured write-only signed storage descriptor was decoded " + "incorrectly"); + const uint64_t mapped_sint_address = base + 0xe0000; + const auto encoded_sint_address = mapped_sint_address >> 8u; + sint_storage.fields[0] = static_cast(encoded_sint_address); + sint_storage.fields[1] = + (sint_storage.fields[1] & ~0xffu) | + static_cast(encoded_sint_address >> 32u); + ShaderRecompiler::IR::DescriptorValue sint_storage_descriptor{}; + std::copy(std::begin(sint_storage.fields), std::end(sint_storage.fields), + sint_storage_descriptor.dwords.begin()); + sint_storage_descriptor.dword_count = 8; + auto sint_storage_resource = srgb_storage_resource; + sint_storage_resource.kind = + ShaderRecompiler::IR::ResourceKind::StorageImageUint; + const auto sint_storage_binding = + RenderExecutorTestAccess::ResolveTexture( + executor, sint_storage_resource, sint_storage_descriptor); + const auto sint_storage_view = texture_cache.FindTexture( + sint_storage_binding.image_id, sint_storage_binding.desc); + Require(name, "PPSA06888 raw R32 SINT storage view", + sint_storage_view != nullptr && + sint_storage_binding.desc.info.data.size == 0x10000 && + sint_storage_binding.desc.info.pixel_format == + vk::Format::eR32Sint && + sint_storage_binding.desc.view_info.format == + vk::Format::eR32Uint && + texture_cache.GetImage(sint_storage_binding.image_id) + .backing.format == vk::Format::eR32Sint, + "write-only R32 SINT storage did not select a bit-compatible " + "uint view"); + + auto narrowed_storage = storage; + constexpr uint64_t narrowed_storage_address = base + 0xd0000; + const auto encoded_narrowed_address = narrowed_storage_address >> 8u; + narrowed_storage.fields[0] = + static_cast(encoded_narrowed_address); + narrowed_storage.fields[1] = + static_cast(encoded_narrowed_address >> 32u) | + (static_cast(stencil_format) << 20u); + narrowed_storage.fields[3] = + DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | + (static_cast(Prospero::ImageType::kColor1DArray) << 28u); + narrowed_storage.fields[4] = 1u | (1u << 16u); + ShaderRecompiler::IR::DescriptorValue narrowed_storage_descriptor{}; + std::copy(std::begin(narrowed_storage.fields), + std::end(narrowed_storage.fields), + narrowed_storage_descriptor.dwords.begin()); + narrowed_storage_descriptor.dword_count = 8; + auto narrowed_storage_resource = storage_resource; + narrowed_storage_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim1D; + const auto narrowed_storage_binding = + RenderExecutorTestAccess::ResolveTexture( + executor, narrowed_storage_resource, narrowed_storage_descriptor); + const auto narrowed_storage_view = texture_cache.FindTexture( + narrowed_storage_binding.image_id, narrowed_storage_binding.desc); + const auto &narrowed_storage_image = + texture_cache.GetImage(narrowed_storage_binding.image_id); + Require(name, "narrowed 1D array storage view", + narrowed_storage_view != nullptr && + narrowed_storage_binding.desc.info.type == + Prospero::ImageType::kColor1D && + narrowed_storage_binding.desc.info.resources.layers == 2 && + narrowed_storage_binding.desc.view_info.type == + vk::ImageViewType::e1D && + narrowed_storage_binding.desc.view_info.base_layer == 1 && + narrowed_storage_binding.desc.view_info.layer_count == 1 && + narrowed_storage_image.backing.image_type == + vk::ImageType::e1D, + "non-array 1D specialization did not select the descriptor " + "base layer from its 1D-array backing"); + + ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{}; + storage_snapshot.images.push_back(storage_descriptor); + ShaderStageRuntime storage_runtime{ + std::make_shared( + std::move(storage_program)), + std::make_shared( + std::move(storage_snapshot))}; + auto storage_discovery = executor.PrepareBindings( + scheduler.Current(), storage_runtime, + vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex); + const auto storage_id = storage_discovery.resources.images[0].image_id; + Require(name, "storage prefetch purity", + storage_discovery.resources.images[0].image_view == nullptr && + texture_cache.GetImage(storage_id).binding.is_bound && + !texture_cache.GetImage(storage_id).usage.storage && + !TextureCacheTestAccess::PendingDownload(texture_cache, + storage_id), + "the first descriptor pass performed final storage acquisition"); + RenderExecutorTestAccess::ResetBindings(executor); + + ShaderRecompiler::IR::Program sampled_program{}; + sampled_program.stage = ShaderType::Pixel; + sampled_program.values = + std::make_shared(); + sampled_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource sampled_resource{}; + sampled_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + sampled_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + sampled_resource.read = true; + sampled_program.info.images.push_back(sampled_resource); + ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{}; + sampled_snapshot.images.push_back(storage_descriptor); + ShaderStageRuntime sampled_runtime{ + std::make_shared( + std::move(sampled_program)), + std::make_shared( + std::move(sampled_snapshot))}; + + constexpr uint64_t ordered_sampled_address = base + 0x10000; + const uint32_t ordered_sampled_value = 0x89abcdefu; + std::memcpy(reinterpret_cast(mapped) + 0x10000, + &ordered_sampled_value, sizeof(ordered_sampled_value)); + auto ordered_sampled = storage; + const auto encoded_ordered_address = ordered_sampled_address >> 8u; + ordered_sampled.fields[0] = + static_cast(encoded_ordered_address); + ordered_sampled.fields[1] = + static_cast(encoded_ordered_address >> 32u) | + (static_cast(stencil_format) << 20u); + ShaderRecompiler::IR::DescriptorValue ordered_descriptor{}; + std::copy(std::begin(ordered_sampled.fields), + std::end(ordered_sampled.fields), + ordered_descriptor.dwords.begin()); + ordered_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{}; + ordered_snapshot.images.push_back(ordered_descriptor); + ShaderStageRuntime ordered_sampled_runtime{ + sampled_runtime.program, + std::make_shared( + std::move(ordered_snapshot))}; + auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( + executor, scheduler.Current(), storage_runtime, + ordered_sampled_runtime, true); + const auto ordered_sampled_id = + ordered_bindings.pixel->resources.images[0].image_id; + Require(name, "VS-before-PS retained-owner order", + ordered_bindings.vertex.resources.images[0].image_id == + storage_id && + ordered_sampled_id != storage_id && + RenderExecutorTestAccess::BoundImagesInOrder( + executor, + TextureCacheTestAccess::Owner(texture_cache, storage_id), + TextureCacheTestAccess::Owner(texture_cache, + ordered_sampled_id)), + "production graphics binding did not retain vertex resources " + "before pixel resources"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto graphics_bindings = + RenderExecutorTestAccess::PrepareGraphicsBindings( + executor, scheduler.Current(), storage_runtime, sampled_runtime, + true); + const auto &storage_binding = + graphics_bindings.vertex.resources.images[0]; + const auto &sampled_binding = + graphics_bindings.pixel->resources.images[0]; + Require(name, "storage final acquisition", + storage_binding.image_view != nullptr && + texture_cache.GetImage(storage_id).usage.storage && + texture_cache.GetImage(storage_id).IsGpuModified() && + TextureCacheTestAccess::PendingDownload(texture_cache, + storage_id), + "the production graphics binding path did not establish GPU " + "ownership before storage readback"); + Require(name, "VS-to-PS image acquisition order", + storage_binding.image_id == storage_id && + sampled_binding.image_id == storage_id && + texture_cache.GetImage(storage_id).IsGpuModified() && + texture_cache.GetImage(storage_id).usage.storage && + texture_cache.GetImage(storage_id).usage.texture, + "the production graphics binding path did not complete vertex " + "storage acquisition before pixel sampling"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto vertex_sampled_program = + std::make_shared( + *sampled_runtime.program); + vertex_sampled_program->stage = ShaderType::Vertex; + ShaderStageRuntime vertex_sampled_runtime{ + std::move(vertex_sampled_program), sampled_runtime.resources}; + auto pixel_storage_program = + std::make_shared( + *storage_runtime.program); + pixel_storage_program->stage = ShaderType::Pixel; + ShaderStageRuntime pixel_storage_runtime{std::move(pixel_storage_program), + storage_runtime.resources}; + auto writable_alias_bindings = + RenderExecutorTestAccess::PrepareGraphicsBindings( + executor, scheduler.Current(), vertex_sampled_runtime, + pixel_storage_runtime, true); + Require(name, "late writable alias promotion", + texture_cache.GetImage(storage_id).binding.force_general, + "an already sampled image was not promoted when a later " + "storage alias bound the same backing"); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + writable_alias_bindings.vertex); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + *writable_alias_bindings.pixel); + Require(name, "forced-general descriptor capture", + writable_alias_bindings.vertex.resources.images[0].layout == + vk::ImageLayout::eGeneral && + writable_alias_bindings.pixel->resources.images[0].layout == + vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + writable_alias_bindings.vertex.resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + writable_alias_bindings.pixel->resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + texture_cache.GetImage(storage_id).backing.state.layout == + vk::ImageLayout::eGeneral, + "sampled/storage aliases did not retain the promoted general " + "layout in both generated descriptors"); + RenderExecutorTestAccess::ResetBindings(executor); + + const std::array split_mip_data{0x01020304u, 0x05060708u, + 0x090a0b0cu}; + std::memcpy(reinterpret_cast(mapped) + 0x50000, + split_mip_data.data(), sizeof(split_mip_data)); + ImageDesc split_desc{}; + split_desc.type = BindingType::Texture; + split_desc.info.data = {base + 0x50000, sizeof(split_mip_data)}; + split_desc.info.pixel_format = vk::Format::eR32Uint; + split_desc.info.guest_format = Prospero::BufferFormat::k32UInt; + split_desc.info.type = Prospero::ImageType::kColor2D; + split_desc.info.extent = {2, 1, 1}; + split_desc.info.resources = {2, 1}; + split_desc.info.pitch = 2; + split_desc.info.bytes_per_block = 4; + split_desc.info.samples = 1; + split_desc.info.tile_mode = linear; + split_desc.info.mip_layout[0] = {0, 8, 2, 1}; + split_desc.info.mip_layout[1] = {8, 4, 1, 1}; + split_desc.view_info.format = split_desc.info.pixel_format; + split_desc.view_info.type = vk::ImageViewType::e2D; + split_desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; + split_desc.view_info.level_count = 2; + split_desc.view_info.layer_count = 1; + split_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto split_id = texture_cache.FindImage(split_desc); + + auto split_storage_desc = split_desc; + split_storage_desc.type = BindingType::Storage; + split_storage_desc.view_info.base_level = 0; + split_storage_desc.view_info.level_count = 1; + split_storage_desc.view_info.usage = vk::ImageUsageFlagBits::eStorage; + auto split_sampled_desc = split_desc; + split_sampled_desc.view_info.base_level = 1; + split_sampled_desc.view_info.level_count = 1; + split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; + + auto split_program = std::make_shared(); + split_program->stage = ShaderType::Vertex; + split_program->values = + std::make_shared(); + split_program->resource_tracking_complete = true; + split_program->info.images = {storage_resource, sampled_resource}; + DescriptorCache::PreparedBindings split_bindings{}; + split_bindings.program = split_program; + split_bindings.snapshot = + std::make_shared(); + split_bindings.shader_stage = vk::ShaderStageFlagBits::eVertex; + split_bindings.stage = DescriptorCache::Stage::Vertex; + split_bindings.resources.images.push_back( + {split_id, texture_cache.FindTexture(split_id, split_storage_desc), + split_storage_desc}); + split_bindings.resources.images.push_back( + {split_id, texture_cache.FindTexture(split_id, split_sampled_desc), + split_sampled_desc}); + RenderExecutorTestAccess::CommitBindings(executor, scheduler.Current(), + split_bindings); + const auto &split_image = texture_cache.GetImage(split_id); + Require(name, "per-binding subresource layouts", + split_bindings.resources.images[0].layout == + vk::ImageLayout::eGeneral && + split_bindings.resources.images[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + DescriptorCacheTestAccess::MakeImageInfo( + split_bindings.resources.images[0]) + .imageLayout == vk::ImageLayout::eGeneral && + DescriptorCacheTestAccess::MakeImageInfo( + split_bindings.resources.images[1]) + .imageLayout == + vk::ImageLayout::eShaderReadOnlyOptimal && + split_image.backing.subresource_states.size() == 2 && + split_image.backing.subresource_states[0].layout == + vk::ImageLayout::eGeneral && + split_image.backing.subresource_states[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "descriptor layouts were queried after a later mip transition " + "instead of being captured at each binding"); + + Libs::LibKernel::Memory::WriteBacking( + storage_address, &storage_stale_value, sizeof(storage_stale_value)); + texture_cache.ProcessDownloadImages(); + Require( + name, "storage acquisition download consumption", + !TextureCacheTestAccess::PendingDownload(texture_cache, storage_id), + "submit-time processing retained an acquired storage request"); + RenderExecutorTestAccess::ResetBindings(executor); + scheduler.FinishCurrent(); + scheduler.DrainPriorityOperations(); + uint32_t storage_downloaded_value = 0; + Libs::LibKernel::Memory::TryReadBacking(storage_address, + &storage_downloaded_value, + sizeof(storage_downloaded_value)); + const uint32_t storage_expected_value = + (storage_stale_value & 0xffffff00u) | (storage_native_value & 0xffu); + Require(name, "storage acquisition readback window", + storage_downloaded_value == storage_expected_value && + texture_cache.GetImage(storage_id).IsGpuModified(), + "readback enrollment was consumed before final storage " + "ownership became visible"); + + constexpr uint64_t phased_depth_address = base + 0x90000; + constexpr uint64_t phased_stencil_address = base + 0xa0000; + constexpr uint64_t phased_htile_address = base + 0xb0000; + HW::DepthRenderTarget phased_depth_target{}; + phased_depth_target.z_info.format = Prospero::DepthFormat::kZ32F; + phased_depth_target.z_info.texture_compatibility = + Prospero::TextureCompatiblePlaneCompression::kEnable; + phased_depth_target.z_info.htile_acceleration = true; + phased_depth_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax; + phased_depth_target.stencil_info.format = Prospero::StencilFormat::k8UInt; + phased_depth_target.stencil_info.texture_compatibility = + Prospero::TextureCompatibleStencil::kEnable; + phased_depth_target.z_read_base_addr = phased_depth_address; + phased_depth_target.z_write_base_addr = phased_depth_address; + phased_depth_target.stencil_read_base_addr = phased_stencil_address; + phased_depth_target.stencil_write_base_addr = phased_stencil_address; + phased_depth_target.htile_data_base_addr = phased_htile_address; + phased_depth_target.size.valid = true; + registers.SetDepthRenderTarget(phased_depth_target); + HW::DepthControl phased_depth_control{}; + phased_depth_control.z_enable = true; + phased_depth_control.z_write_enable = true; + registers.SetDepthControl(phased_depth_control); + HW::RenderControl phased_render_control{}; + registers.SetRenderControl(phased_render_control); + + RenderDepthInfo phased_depth{}; + RenderExecutorTestAccess::ResolveRenderDepthTarget( + executor, 1, scheduler.Current(), phased_depth); + auto non_texture_compatible_target = phased_depth_target; + non_texture_compatible_target.z_info.texture_compatibility = + Prospero::TextureCompatiblePlaneCompression::kDisable; + non_texture_compatible_target.stencil_info.texture_compatibility = + Prospero::TextureCompatibleStencil::kDisable; + registers.SetDepthRenderTarget(non_texture_compatible_target); + RenderDepthInfo non_texture_compatible_depth{}; + RenderExecutorTestAccess::ResolveRenderDepthTarget( + executor, 1, scheduler.Current(), non_texture_compatible_depth); + Require( + name, "depth texture compatibility identity", + phased_depth.image_id && phased_depth.htile && + non_texture_compatible_depth.image_id == phased_depth.image_id && + non_texture_compatible_depth.desc.info.data == + phased_depth.desc.info.data && + non_texture_compatible_depth.desc.info.stencil == + phased_depth.desc.info.stencil && + non_texture_compatible_depth.desc.info.metadata.range == + phased_depth.desc.info.metadata.range && + phased_depth.desc.info.metadata.stencil_compressed && + !phased_depth.depth_clear_enable && + !phased_depth.depth_meta_clear_enable && + !texture_cache.IsMeta(phased_depth.htile_buffer_vaddr) && + !texture_cache.GetImage(phased_depth.image_id).IsGpuModified() && + !texture_cache.GetImage(phased_depth.image_id).usage.depth_target, + "valid PS5 texture-compatibility policy changed logical depth image " + "identity or discovery side effects"); + registers.SetDepthRenderTarget(phased_depth_target); + RenderColorInfo no_color{}; + auto phased_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &no_color, 0, phased_depth); + Require( + name, "HTile final acquisition", + phased_depth.image_view != nullptr && + phased_rendering.num_color_attachments == 0 && + phased_rendering.depth_stencil_attachment.image_view == + phased_depth.image_view && + phased_rendering.depth_stencil_attachment.has_depth && + phased_depth.depth_meta_clear_enable && + phased_depth.depth_load_clear_enable && + texture_cache.IsMeta(phased_depth.htile_buffer_vaddr) && + !texture_cache.IsMetaCleared( + phased_depth.htile_buffer_vaddr, + phased_depth.desc.view_info.base_layer) && + texture_cache.GetImage(phased_depth.image_id).IsGpuModified() && + texture_cache.GetImage(phased_depth.image_id).usage.depth_target, + "final depth acquisition did not publish and consume HTile " + "state at the final acquisition boundary"); + RenderExecutorTestAccess::ResetBindings(executor); + + constexpr uint64_t depth_only_address = base + 0x140000; + HW::DepthRenderTarget depth_only_target{}; + depth_only_target.z_info.format = Prospero::DepthFormat::kZ32F; + depth_only_target.z_info.z_compare_base = Prospero::ZCompareBase::kZMax; + depth_only_target.stencil_info.htile_stencil_disabled = true; + depth_only_target.z_read_base_addr = depth_only_address; + depth_only_target.z_write_base_addr = depth_only_address; + depth_only_target.size = {63, 63, true}; + registers.SetDepthRenderTarget(depth_only_target); + HW::DepthControl depth_only_control{}; + depth_only_control.stencil_enable = true; + depth_only_control.z_enable = true; + depth_only_control.z_write_enable = true; + depth_only_control.zfunc = static_cast(vk::CompareOp::eAlways); + registers.SetDepthControl(depth_only_control); + HW::RenderControl depth_only_render_control{}; + depth_only_render_control.depth_clear_enable = true; + depth_only_render_control.stencil_clear_enable = true; + registers.SetRenderControl(depth_only_render_control); + RenderDepthInfo depth_only{}; + RenderExecutorTestAccess::ResolveRenderDepthTarget( + executor, 1, scheduler.Current(), depth_only); + Require( + name, "depth-only target with stale stencil state", + depth_only.image_id && depth_only.format == vk::Format::eD32Sfloat && + depth_only.depth_test_enable && depth_only.depth_write_enable && + depth_only.depth_clear_enable && + !depth_only.stencil_test_enable && + !depth_only.stencil_clear_enable && + depth_only.stencil_buffer_vaddr == 0 && + depth_only.stencil_buffer_size == 0 && + depth_only.desc.info.stencil.Empty() && + depth_only.depth_buffer_size != 0 && + depth_only_address + depth_only.depth_buffer_size <= + base + allocation_size && + depth_only.vaddr_num == 1 && + depth_only.AttachmentWriteAspects() == + vk::ImageAspectFlagBits::eDepth, + "raw stencil test or clear state leaked into a depth-only " + "attachment"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto video_subresource = + make_target_desc(base + 0x20000, target_mip_size, {1, 1, 1}); + video_subresource.type = BindingType::VideoOut; + video_subresource.info.pixel_format = vk::Format::eR8G8B8A8Srgb; + video_subresource.info.guest_format = + Prospero::BufferFormat::k8_8_8_8Srgb; + video_subresource.view_info.format = video_subresource.info.pixel_format; + video_subresource.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto video_subresource_id = + texture_cache.FindImage(video_subresource); + const auto video_subresource_owner = + TextureCacheTestAccess::Owner(texture_cache, video_subresource_id); + auto video_parent = + make_target_desc(base + 0x20000, target_mip_size * 2, {2, 2, 1}); + video_parent.type = BindingType::VideoOut; + video_parent.info.pixel_format = vk::Format::eR8G8B8A8Srgb; + video_parent.info.guest_format = Prospero::BufferFormat::k8_8_8_8Srgb; + video_parent.info.resources.levels = 2; + video_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; + video_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, + 1}; + video_parent.view_info.format = video_parent.info.pixel_format; + video_parent.view_info.usage = vk::ImageUsageFlagBits::eSampled; + const auto video_parent_id = texture_cache.FindImage(video_parent); + const auto video_parent_owner = + TextureCacheTestAccess::Owner(texture_cache, video_parent_id); + Require(name, "video-out replacement remains cache-only", + video_subresource_owner != nullptr && + video_parent_owner != nullptr && + video_parent_id != video_subresource_id && + !video_subresource_owner->binding.is_bound && + !video_subresource_owner->binding.is_target && + !video_subresource_owner->binding.needs_rebind && + !video_subresource_owner->usage.video_out && + !video_parent_owner->binding.is_bound && + !video_parent_owner->binding.is_target && + !video_parent_owner->binding.needs_rebind && + !video_parent_owner->usage.video_out, + "FindImage leaked caller-owned state across a video-out " + "replacement"); + + constexpr uint64_t array_target_address = base + 0xd0000; + auto array_target = make_target_desc(array_target_address, + target_mip_size, {128, 128, 1}); + const auto array_target_id = texture_cache.FindImage(array_target); + const auto array_target_owner = + TextureCacheTestAccess::Owner(texture_cache, array_target_id); + RenderExecutorTestAccess::BindRenderTarget(executor, array_target_id); + const auto array_target_view = + texture_cache.FindRenderTarget(array_target_id, array_target); + + ShaderTextureResource array_texture{}; + const auto encoded_array_address = array_target_address >> 8u; + constexpr auto array_format = + static_cast(Prospero::BufferFormat::k32UInt); + array_texture.fields[0] = static_cast(encoded_array_address); + array_texture.fields[1] = + static_cast(encoded_array_address >> 32u) | + (array_format << 20u) | (3u << 30u); + array_texture.fields[2] = (127u >> 2u) | (127u << 14u); + array_texture.fields[3] = + DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | + (static_cast(Prospero::ImageType::kColor2DArray) << 28u); + array_texture.fields[4] = 1; + + ShaderRecompiler::IR::Program array_program{}; + array_program.stage = ShaderType::Compute; + array_program.values = + std::make_shared(); + array_program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource array_resource{}; + array_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + array_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + array_resource.read = true; + array_program.info.images.push_back(array_resource); + ShaderRecompiler::IR::DescriptorValue array_descriptor{}; + std::copy(std::begin(array_texture.fields), + std::end(array_texture.fields), + array_descriptor.dwords.begin()); + array_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot array_snapshot{}; + array_snapshot.images.push_back(array_descriptor); + ShaderStageRuntime array_runtime{ + std::make_shared( + std::move(array_program)), + std::make_shared( + std::move(array_snapshot))}; + + auto array_binding = executor.PrepareBindings( + scheduler.Current(), array_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), array_binding); + const auto expanded_array_id = array_binding.resources.images[0].image_id; + const auto &expanded_array = texture_cache.GetImage(expanded_array_id); + Require( + name, "2D target to array backing expansion", + array_target_view != nullptr && array_target_owner != nullptr && + array_target_owner->binding.needs_rebind && + expanded_array_id != array_target_id && + array_binding.resources.images[0].image_view != nullptr && + array_binding.resources.images[0].desc.info.type == + Prospero::ImageType::kColor2D && + array_binding.resources.images[0].desc.info.resources == + ImageSubresources{1, 2} && + array_binding.resources.images[0].desc.view_info.type == + vk::ImageViewType::e2DArray && + array_binding.resources.images[0].desc.view_info.layer_count == + 2 && + expanded_array.backing.layers == 2 && + expanded_array.IsGpuModified() && expanded_array.usage.texture, + "raw array view did not expand and reuse the Color2D backing"); + + RenderColorInfo rebound_array_target{}; + rebound_array_target.type = RenderColorType::RenderTexture; + rebound_array_target.desc = array_target; + rebound_array_target.image_id = array_target_id; + rebound_array_target.format = array_target.view_info.format; + rebound_array_target.extent = {128, 128}; + rebound_array_target.samples = 1; + RenderDepthInfo no_array_depth{}; + auto array_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &rebound_array_target, 1, + no_array_depth); + Require(name, "expanded array target rebind", + rebound_array_target.image_id == expanded_array_id && + rebound_array_target.image_view != nullptr && + array_rendering.num_color_attachments == 1 && + array_rendering.color_attachments[0].image_view == + rebound_array_target.image_view && + array_rendering.width == 128 && + array_rendering.height == 128 && + array_rendering.num_layers == 1 && + texture_cache.GetImage(expanded_array_id).binding.is_bound && + texture_cache.GetImage(expanded_array_id).binding.is_target, + "the retained single-layer target did not rebind to the expanded " + "backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto colliding_msaa_texture = array_texture; + colliding_msaa_texture.fields[3] = + DstSel(4, 5, 6, 7) | (1u << 16u) | + (static_cast(Prospero::TileMode::kRenderTarget) << 20u) | + (static_cast(Prospero::ImageType::kColor2DMsaa) << 28u); + colliding_msaa_texture.fields[4] = 0; + colliding_msaa_texture.fields[5] = (7u << 20u) | (1u << 4u); + ShaderRecompiler::IR::DescriptorValue colliding_msaa_descriptor{}; + std::copy(std::begin(colliding_msaa_texture.fields), + std::end(colliding_msaa_texture.fields), + colliding_msaa_descriptor.dwords.begin()); + colliding_msaa_descriptor.dword_count = 8; + auto colliding_msaa_program = + std::make_shared( + *array_runtime.program); + colliding_msaa_program->info.images[0].dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; + auto colliding_msaa_snapshot = + std::make_shared(); + colliding_msaa_snapshot->images.push_back(colliding_msaa_descriptor); + ShaderStageRuntime colliding_msaa_runtime{ + std::move(colliding_msaa_program), + std::move(colliding_msaa_snapshot)}; + auto colliding_msaa_binding = executor.PrepareBindings( + scheduler.Current(), colliding_msaa_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), colliding_msaa_binding); + const auto &resolved_colliding_msaa = + colliding_msaa_binding.resources.images[0]; + Require( + name, "equal-footprint sample-count identity", + resolved_colliding_msaa.image_id != expanded_array_id && + resolved_colliding_msaa.image_view != nullptr && + resolved_colliding_msaa.desc.info.type == + Prospero::ImageType::kColor2D && + resolved_colliding_msaa.desc.info.resources == + ImageSubresources{1, 1} && + resolved_colliding_msaa.desc.info.samples == 2 && + resolved_colliding_msaa.desc.info.data.size == 0x20000 && + texture_cache.GetImage(resolved_colliding_msaa.image_id) + .backing.samples == 2 && + texture_cache.GetImage(expanded_array_id).backing.samples == 1 && + texture_cache.GetImage(expanded_array_id).backing.layers == 2, + "equal-size 1x array and 2x image shared an incompatible backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + constexpr uint64_t msaa_target_address = base + 0x100000; + constexpr uint32_t msaa_samples = 4; + auto msaa_target = + make_target_desc(msaa_target_address, 0x40000, {128, 128, 1}); + msaa_target.info.samples = msaa_samples; + msaa_target.info.tile_mode = Prospero::TileMode::kRenderTarget; + msaa_target.info.mip_layout[0].size = msaa_target.info.data.size; + const auto msaa_target_id = texture_cache.FindImage(msaa_target); + RenderExecutorTestAccess::BindRenderTarget(executor, msaa_target_id); + const auto msaa_target_view = + texture_cache.FindRenderTarget(msaa_target_id, msaa_target); + + auto msaa_texture = array_texture; + const auto encoded_msaa_address = msaa_target_address >> 8u; + msaa_texture.fields[0] = static_cast(encoded_msaa_address); + msaa_texture.fields[1] = + static_cast(encoded_msaa_address >> 32u) | + (array_format << 20u) | (3u << 30u); + msaa_texture.fields[3] = + DstSel(4, 5, 6, 7) | (2u << 16u) | + (static_cast(Prospero::TileMode::kRenderTarget) << 20u) | + (static_cast(Prospero::ImageType::kColor2DMsaa) << 28u); + msaa_texture.fields[4] = 0; + msaa_texture.fields[5] = (7u << 20u) | (2u << 4u); + ShaderRecompiler::IR::DescriptorValue msaa_descriptor{}; + std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields), + msaa_descriptor.dwords.begin()); + msaa_descriptor.dword_count = 8; + auto msaa_program = std::make_shared( + *array_runtime.program); + msaa_program->info.images[0].dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; + auto msaa_snapshot = + std::make_shared(); + msaa_snapshot->images.push_back(msaa_descriptor); + ShaderStageRuntime msaa_runtime{std::move(msaa_program), + std::move(msaa_snapshot)}; + auto msaa_binding = executor.PrepareBindings( + scheduler.Current(), msaa_runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), msaa_binding); + const auto &resolved_msaa = msaa_binding.resources.images[0]; + Require( + name, "MSAA descriptor backing reuse", + msaa_target_view != nullptr && + resolved_msaa.image_id == msaa_target_id && + resolved_msaa.image_view != nullptr && + resolved_msaa.desc.info.type == Prospero::ImageType::kColor2D && + resolved_msaa.desc.info.resources == ImageSubresources{1, 1} && + resolved_msaa.desc.info.samples == msaa_samples && + resolved_msaa.desc.info.data.size == 0x40000 && + resolved_msaa.desc.view_info.level_count == 1 && + texture_cache.GetImage(msaa_target_id).backing.samples == + msaa_samples, + "MSAA view fields did not resolve to the matching multisample " + "backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto msaa_array_texture = msaa_texture; + msaa_array_texture.fields[3] = + (msaa_array_texture.fields[3] & ~(0xfu << 28u)) | + (static_cast(Prospero::ImageType::kColor2DMsaaArray) + << 28u); + msaa_array_texture.fields[4] = 1; + ShaderRecompiler::IR::DescriptorValue msaa_array_descriptor{}; + std::copy(std::begin(msaa_array_texture.fields), + std::end(msaa_array_texture.fields), + msaa_array_descriptor.dwords.begin()); + msaa_array_descriptor.dword_count = 8; + auto msaa_array_program = std::make_shared( + *msaa_runtime.program); + msaa_array_program->info.images[0].dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray; + auto msaa_array_snapshot = + std::make_shared(); + msaa_array_snapshot->images.push_back(msaa_array_descriptor); + ShaderStageRuntime msaa_array_runtime{std::move(msaa_array_program), + std::move(msaa_array_snapshot)}; + auto msaa_array_binding = executor.PrepareBindings( + scheduler.Current(), msaa_array_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), msaa_array_binding); + const auto &resolved_msaa_array = msaa_array_binding.resources.images[0]; + Require(name, "MSAA array backing expansion", + resolved_msaa_array.image_id != msaa_target_id && + resolved_msaa_array.image_view != nullptr && + resolved_msaa_array.desc.info.type == + Prospero::ImageType::kColor2D && + resolved_msaa_array.desc.info.resources == + ImageSubresources{1, 2} && + resolved_msaa_array.desc.info.samples == msaa_samples && + resolved_msaa_array.desc.info.data.size == 0x80000 && + resolved_msaa_array.desc.view_info.type == + vk::ImageViewType::e2DArray && + resolved_msaa_array.desc.view_info.layer_count == 2 && + texture_cache.GetImage(resolved_msaa_array.image_id) + .backing.layers == 2 && + texture_cache.GetImage(resolved_msaa_array.image_id) + .backing.samples == msaa_samples, + "MSAA array view did not expand the matching Color2D backing"); + RenderExecutorTestAccess::ResetBindings(executor); + + ImageDesc depth{}; + depth.type = BindingType::DepthTarget; + depth.info.data = {depth_address, 0x10000}; + depth.info.stencil = {stencil_address, stencil_layout.size}; + depth.info.pixel_format = vk::Format::eD32SfloatS8Uint; + depth.info.guest_format = Prospero::BufferFormat::k32Float; + depth.info.type = Prospero::ImageType::kColor2D; + depth.info.extent = {1, 1, 1}; + depth.info.resources = {1, 1}; + depth.info.pitch = 1; + depth.info.bytes_per_block = 4; + depth.info.samples = 1; + depth.info.tile_mode = linear; + depth.info.mip_layout[0] = {0, depth.info.data.size, 1, 1}; + depth.view_info.format = depth.info.pixel_format; + depth.view_info.type = vk::ImageViewType::e2D; + depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth; + depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment; + const auto depth_id = texture_cache.FindImage(depth); + Require(name, "cache-only depth discovery", + !texture_cache.FindImageFromRange(stencil_address, + stencil_layout.size, false) && + !texture_cache.GetImage(depth_id).binding.is_bound && + !texture_cache.GetImage(depth_id).binding.is_target && + !texture_cache.GetImage(depth_id).usage.depth_target, + "FindImage created depth acquisition state or a stencil proxy"); + RenderExecutorTestAccess::BindRenderTarget(executor, depth_id); + Require(name, "depth prefetch purity", + texture_cache.GetImage(depth_id).binding.is_target && + !texture_cache.GetImage(depth_id).usage.depth_target && + !texture_cache.FindImageFromRange(stencil_address, + stencil_layout.size, false), + "depth prefetch performed final target acquisition"); + + auto target_parent = + make_target_desc(base, target_mip_size * 2, {2, 2, 1}); + target_parent.type = BindingType::RenderTarget; + target_parent.info.resources.levels = 2; + target_parent.info.mip_layout[0] = {0, target_mip_size, 2, 2}; + target_parent.info.mip_layout[1] = {target_mip_size, target_mip_size, 1, + 1}; + target_parent.view_info.usage = vk::ImageUsageFlagBits::eColorAttachment; + auto target_base_subresource = + make_target_desc(base, target_mip_size, {2, 2, 1}); + const auto target_base_subresource_id = + texture_cache.FindImage(target_base_subresource); + auto target_subresource = + make_target_desc(base + target_mip_size, target_mip_size, {1, 1, 1}); + const auto target_subresource_id = + texture_cache.FindImage(target_subresource); + auto target_subresource_owner = + TextureCacheTestAccess::Owner(texture_cache, target_subresource_id); + std::weak_ptr retired_target = + target_subresource_owner; + const auto &discovered_target = + texture_cache.GetImage(target_subresource_id); + Require(name, "cache-only target discovery", + !discovered_target.binding.is_bound && + !discovered_target.binding.is_target && + !discovered_target.binding.needs_rebind && + !discovered_target.usage.render_target && + !TextureCacheTestAccess::PendingDownload( + texture_cache, target_subresource_id), + "FindImage claimed RenderExecutor-owned render-target state"); + RenderExecutorTestAccess::BindRenderTarget(executor, + target_subresource_id); + Require(name, "target prefetch purity", + texture_cache.GetImage(target_subresource_id).binding.is_target && + !texture_cache.GetImage(target_subresource_id) + .usage.render_target && + !TextureCacheTestAccess::PendingDownload( + texture_cache, target_subresource_id), + "target prefetch performed final target acquisition"); + const auto target_parent_id = texture_cache.FindImage(target_parent); + Require(name, "active target overlap", + target_parent_id && + target_parent_id != target_base_subresource_id && + target_parent_id != target_subresource_id && + target_subresource_owner != nullptr && + target_subresource_owner->binding.needs_rebind && + texture_cache.GetImage(target_parent_id).binding.is_target && + !TextureCacheTestAccess::PendingDownload(texture_cache, + target_parent_id), + "target overlap did not transfer target state to the merged " + "owner"); + target_subresource_owner.reset(); + + RenderColorInfo rebound_color{}; + rebound_color.type = RenderColorType::RenderTexture; + rebound_color.desc = target_subresource; + rebound_color.image_id = target_subresource_id; + rebound_color.format = target_subresource.view_info.format; + rebound_color.extent = {1, 1}; + rebound_color.samples = 1; + RenderDepthInfo rebound_depth{}; + rebound_depth.desc = depth; + rebound_depth.format = depth.info.pixel_format; + rebound_depth.width = 1; + rebound_depth.height = 1; + rebound_depth.samples = 1; + rebound_depth.image_id = depth_id; + scheduler.FinishCurrent(); + auto retired_owner = retired_target.lock(); + Require( + name, "deferred target slot erasure", + TextureCacheTestAccess::Owner(texture_cache, target_subresource_id) == + nullptr && + retired_owner != nullptr && retired_owner->binding.needs_rebind, + "the displaced target did not survive only through RenderExecutor " + "ownership"); + auto rebound_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &rebound_color, 1, rebound_depth); + const auto stencil_proxy_id = texture_cache.FindImageFromRange( + stencil_address, stencil_layout.size, false); + Require( + name, "production target rebind", + rebound_color.image_id == target_parent_id && + rebound_color.image_view != nullptr && + rebound_depth.image_view != nullptr && + rebound_rendering.num_color_attachments == 1 && + rebound_rendering.color_attachments[0].image_view == + rebound_color.image_view && + rebound_rendering.depth_stencil_attachment.image_view == + rebound_depth.image_view && + rebound_rendering.width == 1 && rebound_rendering.height == 1 && + texture_cache.GetImage(target_parent_id).binding.is_target && + texture_cache.GetImage(target_parent_id).usage.render_target && + texture_cache.GetImage(target_parent_id).IsGpuModified() && + texture_cache.GetImage(depth_id).usage.depth_target && + texture_cache.GetImage(depth_id).IsGpuModified() && + TextureCacheTestAccess::PendingDownload(texture_cache, + target_parent_id) && + stencil_proxy_id && + texture_cache.GetImage(stencil_proxy_id).depth_id == depth_id, + "the final target pass did not acquire color before depth and " + "build the dynamic render state"); + scheduler.BeginRendering(rebound_rendering); + scheduler.EndRendering(); + RenderExecutorTestAccess::ResetBindings(executor); + Require(name, "draw-scoped target reset", + !retired_owner->binding.is_target && + !retired_owner->binding.needs_rebind && + !texture_cache.GetImage(target_parent_id).binding.is_target && + !texture_cache.GetImage(depth_id).binding.is_target, + "RenderExecutor retained target binding state after reset"); + retired_owner.reset(); + Require(name, "RenderExecutor-owned retired lifetime", + retired_target.expired(), + "a retired target owner survived after RenderExecutor reset"); + + constexpr uint64_t ordered_color_address = base + 0x80000; + auto ordered_color_desc = + make_target_desc(ordered_color_address, target_mip_size, {1, 1, 1}); + const auto stale_ordered_color = + texture_cache.FindImage(ordered_color_desc); + RenderExecutorTestAccess::BindRenderTarget(executor, stale_ordered_color); + resources.UnmapMemory(ordered_color_address, target_mip_size); + resources.MapMemory(ordered_color_address, target_mip_size); + + auto ordered_depth_desc = depth; + ordered_depth_desc.info.stencil = {ordered_color_address, + target_mip_size}; + const auto ordered_depth_id = texture_cache.FindImage(ordered_depth_desc); + RenderExecutorTestAccess::BindRenderTarget(executor, ordered_depth_id); + RenderColorInfo ordered_color{}; + ordered_color.type = RenderColorType::RenderTexture; + ordered_color.desc = ordered_color_desc; + ordered_color.image_id = stale_ordered_color; + ordered_color.format = ordered_color_desc.view_info.format; + ordered_color.extent = {1, 1}; + ordered_color.samples = 1; + RenderDepthInfo ordered_depth{}; + ordered_depth.desc = ordered_depth_desc; + ordered_depth.format = ordered_depth_desc.info.pixel_format; + ordered_depth.width = 1; + ordered_depth.height = 1; + ordered_depth.samples = 1; + ordered_depth.image_id = ordered_depth_id; + auto ordered_rendering = RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &ordered_color, 1, ordered_depth); + Require(name, "color-before-depth acquisition order", + ordered_color.image_id != stale_ordered_color && + ordered_color.image_view != nullptr && + ordered_depth.image_view != nullptr && + ordered_rendering.color_attachments[0].image_view == + ordered_color.image_view && + ordered_rendering.depth_stencil_attachment.image_view == + ordered_depth.image_view && + texture_cache.GetImage(ordered_color.image_id).depth_id == + ordered_depth.image_id, + "depth acquisition ran before the stale color target was " + "recreated and associated"); + RenderExecutorTestAccess::ResetBindings(executor); + TextureCacheTestAccess::SetLinearReadback(texture_cache, false); + + ShaderTextureResource stencil{}; + const uint64_t encoded_address = stencil_address >> 8u; + stencil.fields[0] = static_cast(encoded_address); + stencil.fields[1] = static_cast(encoded_address >> 32u) | + (static_cast(stencil_format) << 20u); + stencil.fields[3] = + DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + + ShaderRecompiler::IR::Program program{}; + program.stage = ShaderType::Compute; + program.values = std::make_shared(); + program.resource_tracking_complete = true; + ShaderRecompiler::IR::ImageResource stencil_resource{}; + stencil_resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + stencil_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + stencil_resource.read = true; + program.info.images.push_back(stencil_resource); + + ShaderRecompiler::IR::ResourceSnapshot snapshot{}; + ShaderRecompiler::IR::DescriptorValue descriptor{}; + std::copy(std::begin(stencil.fields), std::end(stencil.fields), + descriptor.dwords.begin()); + descriptor.dword_count = 8; + snapshot.images.push_back(descriptor); + ShaderStageRuntime runtime{ + std::make_shared( + std::move(program)), + std::make_shared( + std::move(snapshot))}; + + auto prepared = context.GetRenderExecutor().PrepareBindings( + scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + const auto sampled_stencil_id = prepared.resources.images[0].image_id; + Require(name, "first stencil discovery", + prepared.resources.images.size() == 1 && + sampled_stencil_id != depth_id && + prepared.resources.images[0].image_view == nullptr && + texture_cache.GetImage(sampled_stencil_id).binding.is_bound && + !texture_cache.GetImage(stencil_proxy_id).binding.is_bound && + !texture_cache.GetImage(stencil_proxy_id).binding.is_target, + "the first sampled-stencil lookup did not remain an ordinary " + "discovery before final depth acquisition"); + + context.GetRenderExecutor().RebindImages(scheduler.Current(), prepared); + Require(name, "first stencil acquisition", + prepared.resources.images[0].image_id == sampled_stencil_id && + prepared.resources.images[0].image_view != nullptr && + texture_cache.GetImage(sampled_stencil_id).usage.texture, + "the first sampled-stencil image was not finally acquired"); + + RenderExecutorTestAccess::ResetBindings(executor); + RenderDepthInfo reassociated_depth = rebound_depth; + reassociated_depth.image_view = nullptr; + RenderColorInfo no_reassociated_color{}; + (void)RenderExecutorTestAccess::AcquireRenderTargets( + executor, scheduler.Current(), &no_reassociated_color, 0, + reassociated_depth); + Require(name, "existing stencil association selection", + texture_cache.GetImage(sampled_stencil_id).depth_id == + reassociated_depth.image_id, + "final depth acquisition did not associate the existing image " + "at the stencil guest address"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto redirected = context.GetRenderExecutor().PrepareBindings( + scheduler.Current(), runtime, vk::ShaderStageFlagBits::eCompute, + DescriptorCache::Stage::Compute); + Require(name, "redirected owner discovery", + redirected.resources.images.size() == 1 && + redirected.resources.images[0].image_id == depth_id && + redirected.resources.images[0].image_view == nullptr && + texture_cache.GetImage(depth_id).binding.is_bound && + !texture_cache.GetImage(sampled_stencil_id).binding.is_bound, + "the established stencil association did not redirect the next " + "discovery to the depth owner"); + context.GetRenderExecutor().RebindImages(scheduler.Current(), redirected); + Require(name, "second-pass stencil acquisition", + redirected.resources.images[0].image_id == depth_id && + redirected.resources.images[0].image_view != nullptr && + texture_cache.GetImage(depth_id).usage.texture, + "RebindImages did not acquire the associated depth owner"); + RenderExecutorTestAccess::ResetBindings(executor); + + auto stencil_storage_program = + std::make_shared(*runtime.program); + stencil_storage_program->info.images[0].kind = + ShaderRecompiler::IR::ResourceKind::StorageImageUint; + stencil_storage_program->info.images[0].read = false; + stencil_storage_program->info.images[0].written = true; + auto storage_stencil = stencil; + storage_stencil.fields[5] = 0x00700000u; + ShaderRecompiler::IR::DescriptorValue stencil_storage_descriptor{}; + std::copy(std::begin(storage_stencil.fields), + std::end(storage_stencil.fields), + stencil_storage_descriptor.dwords.begin()); + stencil_storage_descriptor.dword_count = 8; + ShaderRecompiler::IR::ResourceSnapshot stencil_storage_snapshot{}; + stencil_storage_snapshot.images.push_back(stencil_storage_descriptor); + ShaderStageRuntime stencil_storage_runtime{ + std::move(stencil_storage_program), + std::make_shared( + std::move(stencil_storage_snapshot))}; + auto storage_redirected = executor.PrepareBindings( + scheduler.Current(), stencil_storage_runtime, + vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute); + executor.RebindImages(scheduler.Current(), storage_redirected); + Require( + name, "storage stencil acquisition", + storage_redirected.resources.images[0].image_id == depth_id && + storage_redirected.resources.images[0].image_view != nullptr && + texture_cache.GetImage(depth_id).usage.storage, + "storage stencil binding did not acquire the associated depth owner"); + RenderExecutorTestAccess::ResetBindings(executor); + resources.UnmapMemory(base, allocation_size); + scheduler.Finish(); + } + + Require(name, "unmap direct backing", + Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0, + "descriptor discovery direct mapping release failed"); + Require(name, "release direct backing", + Libs::LibKernel::Memory::KernelReleaseDirectMemory( + direct_offset, allocation_size) == 0, + "descriptor discovery direct-memory allocation release failed"); + std::printf("[host] %-32s ok\n", name); + } + + Buffer CreateStorageBuffer(const char *shader_name, + const std::vector &initial, + size_t dword_count) { + Buffer ret; + ret.size = static_cast(std::max(dword_count, 1u) * + sizeof(u32)); + + vk::BufferCreateInfo buffer_info{}; + buffer_info.sType = vk::StructureType::eBufferCreateInfo; + buffer_info.size = ret.size; + buffer_info.usage = vk::BufferUsageFlagBits::eStorageBuffer; + buffer_info.sharingMode = vk::SharingMode::eExclusive; + RequireVk(shader_name, "dispatch", + m_device.createBuffer(&buffer_info, nullptr, &ret.buffer), + "vkCreateBuffer"); + + vk::MemoryRequirements req{}; + m_device.getBufferMemoryRequirements(ret.buffer, &req); + u32 memory_type = 0; + ret.coherent = FindMemoryType(req.memoryTypeBits, + vk::MemoryPropertyFlagBits::eHostVisible | + vk::MemoryPropertyFlagBits::eHostCoherent, + &memory_type); + if (!ret.coherent) { + Require(shader_name, "dispatch", + FindMemoryType(req.memoryTypeBits, + vk::MemoryPropertyFlagBits::eHostVisible, + &memory_type), + "no host-visible memory type for storage buffer"); + } + + vk::MemoryAllocateInfo alloc{}; + alloc.sType = vk::StructureType::eMemoryAllocateInfo; + alloc.allocationSize = req.size; + alloc.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + m_device.allocateMemory(&alloc, nullptr, &ret.memory), + "vkAllocateMemory"); + RequireVk(shader_name, "dispatch", + m_device.bindBufferMemory(ret.buffer, ret.memory, 0), + "vkBindBufferMemory"); + + std::vector contents(dword_count, 0); + for (size_t i = 0; i < initial.size() && i < contents.size(); i++) { + contents[i] = initial[i]; + } + WriteBuffer(shader_name, ret, contents); + return ret; + } + + void DestroyBuffer(Buffer *buffer) { + if (buffer == nullptr) { + return; + } + if (buffer->buffer != nullptr) { + m_device.destroyBuffer(buffer->buffer, nullptr); + buffer->buffer = nullptr; + } + if (buffer->memory != nullptr) { + m_device.freeMemory(buffer->memory, nullptr); + buffer->memory = nullptr; + } + } + + Image CreateImage2D(const char *shader_name, u32 width, u32 height, + vk::Format format, vk::ImageUsageFlags usage, + const std::vector &initial, u32 dwords_per_pixel, + vk::ImageLayout final_layout) { + std::vector> mips; + if (!initial.empty()) { + mips.push_back(initial); + } + return CreateImageMips(shader_name, width, height, format, usage, mips, + dwords_per_pixel, final_layout, vk::ImageType::e2D, + vk::ImageViewType::e2D, 1); + } + + static u32 MipExtent(u32 value, u32 level) { + for (u32 i = 0; i < level && value > 1u; i++) { + value >>= 1u; + } + return std::max(value, 1u); + } + + static size_t ImageMipDwordCount(u32 width, u32 height, u32 dwords_per_pixel, + u32 level, u32 layers = 1) { + return static_cast(MipExtent(width, level)) * + static_cast(MipExtent(height, level)) * layers * + dwords_per_pixel; + } + + Image CreateImageMips(const char *shader_name, u32 width, u32 height, + vk::Format format, vk::ImageUsageFlags usage, + const std::vector> &initial_mips, + u32 dwords_per_pixel, vk::ImageLayout final_layout, + vk::ImageType image_type, vk::ImageViewType view_type, + u32 layers, u32 view_base_layer = 0, + u32 view_layers = 0) { + Image ret; + ret.format = format; + ret.width = width; + ret.height = height; + ret.layers = layers; + ret.mip_levels = std::max(static_cast(initial_mips.size()), 1u); + ret.dwords_per_pixel = dwords_per_pixel; + + vk::ImageCreateInfo image_info{}; + image_info.sType = vk::StructureType::eImageCreateInfo; + image_info.imageType = image_type; + image_info.format = format; + image_info.extent.width = width; + image_info.extent.height = height; + image_info.extent.depth = 1; + image_info.mipLevels = ret.mip_levels; + image_info.arrayLayers = layers; + image_info.samples = vk::SampleCountFlagBits::e1; + image_info.tiling = vk::ImageTiling::eOptimal; + image_info.usage = usage | vk::ImageUsageFlagBits::eTransferDst | + vk::ImageUsageFlagBits::eTransferSrc; + image_info.sharingMode = vk::SharingMode::eExclusive; + image_info.initialLayout = vk::ImageLayout::eUndefined; + RequireVk(shader_name, "dispatch", + m_device.createImage(&image_info, nullptr, &ret.image), + "vkCreateImage"); + + vk::MemoryRequirements req{}; + m_device.getImageMemoryRequirements(ret.image, &req); + u32 memory_type = 0; + if (!FindMemoryType(req.memoryTypeBits, + vk::MemoryPropertyFlagBits::eDeviceLocal, + &memory_type)) { + Require(shader_name, "dispatch", + FindMemoryType(req.memoryTypeBits, {}, &memory_type), + "no memory type for image"); + } + + vk::MemoryAllocateInfo alloc{}; + alloc.sType = vk::StructureType::eMemoryAllocateInfo; + alloc.allocationSize = req.size; + alloc.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + m_device.allocateMemory(&alloc, nullptr, &ret.memory), + "vk::Device::allocateMemory(image)"); + RequireVk(shader_name, "dispatch", + m_device.bindImageMemory(ret.image, ret.memory, 0), + "vkBindImageMemory"); + + vk::ImageViewCreateInfo view_info{}; + view_info.sType = vk::StructureType::eImageViewCreateInfo; + view_info.image = ret.image; + view_info.viewType = view_type; + view_info.format = format; + view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + view_info.subresourceRange.baseMipLevel = 0; + view_info.subresourceRange.levelCount = ret.mip_levels; + Require(shader_name, "dispatch", view_base_layer < layers, + "image view base layer is out of bounds"); + if (view_layers == 0) { + view_layers = layers - view_base_layer; + } + Require(shader_name, "dispatch", view_layers <= layers - view_base_layer, + "image view layer count is out of bounds"); + view_info.subresourceRange.baseArrayLayer = view_base_layer; + view_info.subresourceRange.layerCount = view_layers; + RequireVk(shader_name, "dispatch", + m_device.createImageView(&view_info, nullptr, &ret.view), + "vkCreateImageView"); + + if (!initial_mips.empty()) { + size_t total_dwords = 0; + for (u32 level = 0; level < ret.mip_levels; level++) { + total_dwords += + ImageMipDwordCount(width, height, dwords_per_pixel, level, layers); + } + std::vector contents(total_dwords, 0); + size_t offset = 0; + for (u32 level = 0; level < ret.mip_levels; level++) { + const auto level_dwords = + ImageMipDwordCount(width, height, dwords_per_pixel, level, layers); + const auto &src = initial_mips[level]; + for (size_t i = 0; i < src.size() && i < level_dwords; i++) { + contents[offset + i] = src[i]; + } + offset += level_dwords; + } + auto staging = + CreateHostBuffer(shader_name, contents.size() * sizeof(u32), + vk::BufferUsageFlagBits::eTransferSrc, contents); + UploadImageMips(shader_name, &ret, staging.buffer, final_layout); + DestroyBuffer(&staging); + } else { + TransitionImage(shader_name, &ret, final_layout, + vk::PipelineStageFlagBits::eTopOfPipe, + vk::PipelineStageFlagBits::eComputeShader, {}, + AccessForLayout(final_layout)); + } + return ret; + } + + void DestroyImage(Image *image) { + if (image == nullptr) { + return; + } + if (image->view != nullptr) { + m_device.destroyImageView(image->view, nullptr); + image->view = nullptr; + } + if (image->image != nullptr) { + m_device.destroyImage(image->image, nullptr); + image->image = nullptr; + } + if (image->memory != nullptr) { + m_device.freeMemory(image->memory, nullptr); + image->memory = nullptr; + } + } + + std::vector ReadImage(const char *shader_name, Image *image) { + const auto dword_count = static_cast(image->width) * + static_cast(image->height) * + image->layers * image->dwords_per_pixel; + auto staging = CreateHostBuffer(shader_name, dword_count * sizeof(u32), + vk::BufferUsageFlagBits::eTransferDst, {}); + + vk::CommandBuffer cmd = BeginCommands(shader_name, "readback"); + AddImageBarrier( + cmd, image->image, image->layout, vk::ImageLayout::eTransferSrcOptimal, + vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite, + vk::AccessFlagBits::eTransferRead, image->mip_levels, image->layers); + vk::BufferImageCopy copy{}; + copy.bufferOffset = 0; + copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + copy.imageSubresource.mipLevel = 0; + copy.imageSubresource.baseArrayLayer = 0; + copy.imageSubresource.layerCount = image->layers; + copy.imageExtent.width = image->width; + copy.imageExtent.height = image->height; + copy.imageExtent.depth = 1; + cmd.copyImageToBuffer(image->image, vk::ImageLayout::eTransferSrcOptimal, + staging.buffer, 1, ©); + EndSubmitAndFree(shader_name, "readback", cmd); + image->layout = vk::ImageLayout::eTransferSrcOptimal; + + auto ret = ReadBuffer(shader_name, staging, dword_count); + DestroyBuffer(&staging); + return ret; + } + + vk::Sampler CreateNearestSampler(const char *shader_name) { + vk::SamplerCreateInfo sampler_info{}; + sampler_info.sType = vk::StructureType::eSamplerCreateInfo; + sampler_info.magFilter = vk::Filter::eNearest; + sampler_info.minFilter = vk::Filter::eNearest; + sampler_info.mipmapMode = vk::SamplerMipmapMode::eNearest; + sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge; + sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge; + sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; + sampler_info.minLod = 0.0f; + sampler_info.maxLod = 0.0f; + vk::Sampler sampler = nullptr; + RequireVk(shader_name, "dispatch", + m_device.createSampler(&sampler_info, nullptr, &sampler), + "vkCreateSampler"); + return sampler; + } + + std::vector ReadBuffer(const char *shader_name, const Buffer &buffer, + size_t dword_count) { + if (!buffer.coherent) { + vk::MappedMemoryRange range{}; + range.sType = vk::StructureType::eMappedMemoryRange; + range.memory = buffer.memory; + range.offset = 0; + range.size = VK_WHOLE_SIZE; + RequireVk(shader_name, "readback", + m_device.invalidateMappedMemoryRanges(1, &range), + "vkInvalidateMappedMemoryRanges"); + } + + void *data = nullptr; + RequireVk(shader_name, "readback", + m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data), + "vkMapMemory"); + std::vector ret(dword_count, 0); + std::memcpy(ret.data(), data, dword_count * sizeof(u32)); + m_device.unmapMemory(buffer.memory); + return ret; + } + + void Dispatch(const TestCase &test, const CompiledShader &compiled, + const Buffer &buffer, const Buffer *gds_buffer = nullptr, + const Image *sampled_image = nullptr, + const Image *storage_image = nullptr, + const Image *storage_image_uint = nullptr, + vk::Sampler sampler = nullptr) { + using Kind = ShaderRecompiler::IR::DescriptorBindingKind; + const auto &layout = compiled.program.bindings; + auto Binding = [&](Kind kind) { + return ShaderRecompiler::IR::FindBinding(layout, kind); + }; + auto Count = [&](Kind kind) { + const auto *binding = Binding(kind); + if (binding == nullptr) { + return 0u; + } + if (kind == Kind::Gds) { + return 1u; + } + return static_cast(binding->resources.size()); + }; + Require( + test.name, "dispatch", + Count(Kind::Sampled2DArray) == 0 && Count(Kind::Sampled3D) == 0 && + Count(Kind::SampledUint2DArray) == 0 && + Count(Kind::SampledUint3D) == 0 && Count(Kind::Storage1D) == 0 && + Count(Kind::Storage1DArray) == 0 && + Count(Kind::Storage2DArray) == 0 && Count(Kind::Storage3D) == 0 && + Count(Kind::StorageUint1D) == 0 && + Count(Kind::StorageUint1DArray) == 0 && + Count(Kind::StorageUint2DArray) == 0 && + Count(Kind::StorageUint3D) == 0, + "unsupported array/3D image cases must provide matching Vulkan test " + "views " + "before dispatch"); + + vk::ShaderModuleCreateInfo module_info{}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = compiled.spirv.size() * sizeof(u32); + module_info.pCode = compiled.spirv.data(); + vk::ShaderModule module = nullptr; + RequireVk(test.name, "SPIR-V module", + m_device.createShaderModule(&module_info, nullptr, &module), + "vkCreateShaderModule"); + + std::vector layout_bindings; + auto add_layout_binding = + [&layout_bindings](u32 binding, vk::DescriptorType type, u32 count) { + if (count == 0) { + return; + } + vk::DescriptorSetLayoutBinding item{}; + item.binding = binding; + item.descriptorType = type; + item.descriptorCount = count; + item.stageFlags = vk::ShaderStageFlagBits::eCompute; + layout_bindings.push_back(item); + }; + for (const auto &binding : layout.descriptors) { + vk::DescriptorType type = vk::DescriptorType::eStorageBuffer; + u32 count = static_cast(binding.resources.size()); + switch (binding.kind) { + case Kind::Sampled1D: + case Kind::Sampled1DArray: + case Kind::Sampled2D: + case Kind::Sampled2DArray: + case Kind::Sampled3D: + case Kind::SampledUint1D: + case Kind::SampledUint1DArray: + case Kind::SampledUint2D: + case Kind::SampledUint2DArray: + case Kind::SampledUint3D: + type = vk::DescriptorType::eSampledImage; + break; + case Kind::Storage1D: + case Kind::Storage1DArray: + case Kind::Storage2D: + case Kind::Storage2DArray: + case Kind::Storage3D: + case Kind::StorageUint1D: + case Kind::StorageUint1DArray: + case Kind::StorageUint2D: + case Kind::StorageUint2DArray: + case Kind::StorageUint3D: + type = vk::DescriptorType::eStorageImage; + break; + case Kind::Samplers: + type = vk::DescriptorType::eSampler; + break; + case Kind::Gds: + case Kind::FlattenedSrt: + case Kind::UserData: + count = 1; + break; + default: + break; + } + add_layout_binding(binding.binding, type, count); + } + + vk::DescriptorSetLayoutCreateInfo layout_info{}; + layout_info.sType = vk::StructureType::eDescriptorSetLayoutCreateInfo; + layout_info.bindingCount = static_cast(layout_bindings.size()); + layout_info.pBindings = + layout_bindings.empty() ? nullptr : layout_bindings.data(); + vk::DescriptorSetLayout descriptor_layout = nullptr; + RequireVk(test.name, "dispatch", + m_device.createDescriptorSetLayout(&layout_info, nullptr, + &descriptor_layout), + "vkCreateDescriptorSetLayout"); + + vk::PipelineLayoutCreateInfo pipeline_layout_info{}; + pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + pipeline_layout_info.setLayoutCount = 1; + pipeline_layout_info.pSetLayouts = &descriptor_layout; + vk::PushConstantRange push_range{}; + if (layout.push_constant_size != 0) { + push_range.stageFlags = vk::ShaderStageFlagBits::eCompute; + push_range.offset = layout.push_constant_offset; + push_range.size = layout.push_constant_size; + pipeline_layout_info.pushConstantRangeCount = 1; + pipeline_layout_info.pPushConstantRanges = &push_range; + } + vk::PipelineLayout pipeline_layout = nullptr; + RequireVk(test.name, "dispatch", + m_device.createPipelineLayout(&pipeline_layout_info, nullptr, + &pipeline_layout), + "vkCreatePipelineLayout"); + + vk::PipelineShaderStageCreateInfo stage{}; + stage.sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stage.stage = vk::ShaderStageFlagBits::eCompute; + stage.module = module; + stage.pName = "main"; + + vk::ComputePipelineCreateInfo pipeline_info{}; + pipeline_info.sType = vk::StructureType::eComputePipelineCreateInfo; + pipeline_info.stage = stage; + pipeline_info.layout = pipeline_layout; + vk::Pipeline pipeline = nullptr; + RequireVk(test.name, "dispatch", + m_device.createComputePipelines(nullptr, 1, &pipeline_info, + nullptr, &pipeline), + "vkCreateComputePipelines"); + + std::vector pool_sizes; + auto add_pool_size = [&pool_sizes](vk::DescriptorType type, u32 count) { + if (count == 0) { + return; + } + for (auto &size : pool_sizes) { + if (size.type == type) { + size.descriptorCount += count; + return; + } + } + pool_sizes.push_back({type, count}); + }; + add_pool_size(vk::DescriptorType::eStorageBuffer, + Count(Kind::Buffers) + Count(Kind::AddressMemory) + + Count(Kind::Gds) + + (Binding(Kind::FlattenedSrt) != nullptr ? 1u : 0u) + + (Binding(Kind::UserData) != nullptr ? 1u : 0u)); + add_pool_size( + vk::DescriptorType::eSampledImage, + Count(Kind::Sampled1D) + Count(Kind::Sampled1DArray) + + Count(Kind::Sampled2D) + Count(Kind::Sampled2DArray) + + Count(Kind::Sampled3D) + Count(Kind::SampledUint1D) + + Count(Kind::SampledUint1DArray) + Count(Kind::SampledUint2D) + + Count(Kind::SampledUint2DArray) + Count(Kind::SampledUint3D)); + add_pool_size( + vk::DescriptorType::eStorageImage, + Count(Kind::Storage1D) + Count(Kind::Storage1DArray) + + Count(Kind::Storage2D) + Count(Kind::Storage2DArray) + + Count(Kind::Storage3D) + Count(Kind::StorageUint1D) + + Count(Kind::StorageUint1DArray) + Count(Kind::StorageUint2D) + + Count(Kind::StorageUint2DArray) + Count(Kind::StorageUint3D)); + add_pool_size(vk::DescriptorType::eSampler, Count(Kind::Samplers)); + vk::DescriptorPoolCreateInfo pool_info{}; + pool_info.sType = vk::StructureType::eDescriptorPoolCreateInfo; + pool_info.maxSets = 1; + pool_info.poolSizeCount = static_cast(pool_sizes.size()); + pool_info.pPoolSizes = pool_sizes.empty() ? nullptr : pool_sizes.data(); + vk::DescriptorPool descriptor_pool = nullptr; + RequireVk( + test.name, "dispatch", + m_device.createDescriptorPool(&pool_info, nullptr, &descriptor_pool), + "vkCreateDescriptorPool"); + + vk::DescriptorSetAllocateInfo set_info{}; + set_info.sType = vk::StructureType::eDescriptorSetAllocateInfo; + set_info.descriptorPool = descriptor_pool; + set_info.descriptorSetCount = 1; + set_info.pSetLayouts = &descriptor_layout; + vk::DescriptorSet descriptor_set = nullptr; + RequireVk(test.name, "dispatch", + m_device.allocateDescriptorSets(&set_info, &descriptor_set), + "vkAllocateDescriptorSets"); + + std::vector writes; + std::vector buffer_infos; + std::vector address_memory_infos; + std::vector sampled_infos; + std::vector storage_infos; + std::vector storage_uint_infos; + std::vector sampler_infos; + Buffer flattened_buffer; + Buffer user_data_buffer; + vk::DescriptorBufferInfo flattened_info{}; + vk::DescriptorBufferInfo user_data_info{}; + vk::DescriptorBufferInfo gds_info{}; + + const auto *buffers = Binding(Kind::Buffers); + if (buffers != nullptr) { + buffer_infos.resize(buffers->resources.size()); + for (u32 i = 0; i < buffer_infos.size(); i++) { + auto &info = buffer_infos[i]; + info.buffer = buffer.buffer; + info.offset = 0; + info.range = buffer.size; + if (test.storage_buffer_range_dwords != 0) { + const auto offset = i < test.storage_buffer_offsets.size() + ? test.storage_buffer_offsets[i] + : 0u; + info.range = static_cast( + test.storage_buffer_range_dwords * sizeof(u32) + offset); + Require(test.name, "dispatch", info.range <= buffer.size, + "storage buffer descriptor range exceeds backing buffer"); + } + } + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = buffers->binding; + write.descriptorCount = static_cast(buffer_infos.size()); + write.descriptorType = vk::DescriptorType::eStorageBuffer; + write.pBufferInfo = buffer_infos.data(); + writes.push_back(write); + } + if (const auto *address = Binding(Kind::AddressMemory); + address != nullptr) { + address_memory_infos.resize(address->resources.size()); + for (auto &info : address_memory_infos) { + info = {buffer.buffer, 0, buffer.size}; + } + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = address->binding; + write.descriptorCount = static_cast(address_memory_infos.size()); + write.descriptorType = vk::DescriptorType::eStorageBuffer; + write.pBufferInfo = address_memory_infos.data(); + writes.push_back(write); + } + if (const auto *flattened = Binding(Kind::FlattenedSrt); + flattened != nullptr) { + flattened_buffer = CreateStorageBuffer(test.name, compiled.flattened_srt, + compiled.flattened_srt.size()); + flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size}; + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = flattened->binding; + write.descriptorCount = 1; + write.descriptorType = vk::DescriptorType::eStorageBuffer; + write.pBufferInfo = &flattened_info; + writes.push_back(write); + } + if (const auto *user = Binding(Kind::UserData); user != nullptr) { + user_data_buffer = + CreateStorageBuffer(test.name, compiled.packed_user_data, + compiled.packed_user_data.size()); + user_data_info = {user_data_buffer.buffer, 0, user_data_buffer.size}; + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = user->binding; + write.descriptorCount = 1; + write.descriptorType = vk::DescriptorType::eStorageBuffer; + write.pBufferInfo = &user_data_info; + writes.push_back(write); + } + if (const auto *gds = Binding(Kind::Gds); gds != nullptr) { + Require(test.name, "dispatch", gds_buffer != nullptr, + "GDS descriptor requested but no GDS buffer was provided"); + gds_info = {gds_buffer->buffer, 0, gds_buffer->size}; + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = gds->binding; + write.descriptorCount = 1; + write.descriptorType = vk::DescriptorType::eStorageBuffer; + write.pBufferInfo = &gds_info; + writes.push_back(write); + } + const ShaderRecompiler::IR::DescriptorBinding *sampled = nullptr; + constexpr std::array sampled_kinds{ + Kind::Sampled1D, Kind::Sampled1DArray, Kind::Sampled2D, + Kind::SampledUint1D, Kind::SampledUint1DArray, Kind::SampledUint2D, + }; + for (const auto kind : sampled_kinds) { + if (const auto *candidate = Binding(kind); candidate != nullptr) { + Require(test.name, "dispatch", sampled == nullptr, + "Vulkan test harness needs separate sampled images for mixed " + "descriptor classes"); + sampled = candidate; + } + } + if (sampled != nullptr) { + Require(test.name, "dispatch", sampled_image != nullptr, + "sampled image descriptor requested but no sampled image was " + "provided"); + sampled_infos.resize(sampled->resources.size()); + for (auto &info : sampled_infos) { + info.imageView = sampled_image->view; + info.imageLayout = sampled_image->layout; + } + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = sampled->binding; + write.descriptorCount = static_cast(sampled_infos.size()); + write.descriptorType = vk::DescriptorType::eSampledImage; + write.pImageInfo = sampled_infos.data(); + writes.push_back(write); + } + const auto *storage = Binding(Kind::Storage2D); + const auto *storage_uint = Binding(Kind::StorageUint2D); + if (storage != nullptr || storage_uint != nullptr) { + Require(test.name, "dispatch", storage_image != nullptr, + "storage image descriptor requested but no storage image was " + "provided"); + Require(test.name, "dispatch", storage_image_uint != nullptr, + "uint storage image descriptor requested but no uint storage " + "image was provided"); + storage_infos.resize(storage != nullptr ? storage->resources.size() : 0u); + storage_uint_infos.resize( + storage_uint != nullptr ? storage_uint->resources.size() : 0u); + for (auto &info : storage_infos) { + info.imageView = storage_image->view; + info.imageLayout = storage_image->layout; + } + for (auto &info : storage_uint_infos) { + info.imageView = storage_image_uint->view; + info.imageLayout = storage_image_uint->layout; + } + if (storage != nullptr) { + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = storage->binding; + write.descriptorCount = static_cast(storage_infos.size()); + write.descriptorType = vk::DescriptorType::eStorageImage; + write.pImageInfo = storage_infos.data(); + writes.push_back(write); + } + if (storage_uint != nullptr) { + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = storage_uint->binding; + write.descriptorCount = static_cast(storage_uint_infos.size()); + write.descriptorType = vk::DescriptorType::eStorageImage; + write.pImageInfo = storage_uint_infos.data(); + writes.push_back(write); + } + } + const auto *samplers = Binding(Kind::Samplers); + if (samplers != nullptr) { + Require(test.name, "dispatch", sampler != nullptr, + "sampler descriptor requested but no sampler was provided"); + sampler_infos.resize(samplers->resources.size()); + for (auto &info : sampler_infos) { + info.sampler = sampler; + } + vk::WriteDescriptorSet write{}; + write.sType = vk::StructureType::eWriteDescriptorSet; + write.dstSet = descriptor_set; + write.dstBinding = samplers->binding; + write.descriptorCount = static_cast(sampler_infos.size()); + write.descriptorType = vk::DescriptorType::eSampler; + write.pImageInfo = sampler_infos.data(); + writes.push_back(write); + } + if (!writes.empty()) { + m_device.updateDescriptorSets(static_cast(writes.size()), + writes.data(), 0, nullptr); + } + + vk::CommandBuffer cmd = BeginCommands(test.name, "dispatch"); + cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline); + cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0, + 1, &descriptor_set, 0, nullptr); + if (layout.push_constant_size != 0) { + Require(test.name, "dispatch", + compiled.packed_user_data.size() * sizeof(u32) == + layout.push_constant_size, + "native user-data size does not match push-constant range"); + cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, + layout.push_constant_offset, layout.push_constant_size, + compiled.packed_user_data.data()); + } + cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z); + + if (buffers != nullptr) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = + vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer.buffer; + barrier.offset = 0; + barrier.size = buffer.size; + cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &barrier, 0, nullptr); + } + if (gds_buffer != nullptr) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = + vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = gds_buffer->buffer; + barrier.offset = 0; + barrier.size = gds_buffer->size; + cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader, + vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &barrier, 0, nullptr); + } + EndSubmitAndFree(test.name, "dispatch", cmd); + if (flattened_buffer.buffer != nullptr) { + DestroyBuffer(&flattened_buffer); + } + if (user_data_buffer.buffer != nullptr) { + DestroyBuffer(&user_data_buffer); + } + m_device.destroyDescriptorPool(descriptor_pool, nullptr); + m_device.destroyPipeline(pipeline, nullptr); + m_device.destroyPipelineLayout(pipeline_layout, nullptr); + m_device.destroyDescriptorSetLayout(descriptor_layout, nullptr); + m_device.destroyShaderModule(module, nullptr); + } + + std::vector RenderFragment(const GraphicsCase &test, + const CompiledShader &fragment) { + const auto vertex_spirv = TestSpv::MakePassthroughVertexSpirv(); + ValidateSpirv(test.name, vertex_spirv); + + Image target = + CreateImage2D(test.name, 1, 1, vk::Format::eR32G32B32A32Sfloat, + vk::ImageUsageFlagBits::eColorAttachment, {}, 4, + vk::ImageLayout::eGeneral); + const std::vector default_vertices = { + 0xbf800000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, 0x3f400000u, + 0x3f800000u, 0x40400000u, 0xbf800000u, 0x3e800000u, 0x3f000000u, + 0x3f400000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u, + 0x3f000000u, 0x3f400000u, 0x3f800000u, + }; + const auto &vertices = + test.vertices.empty() ? default_vertices : test.vertices; + Require(test.name, "graphics", vertices.size() == 18u, + "graphics vertex buffer must contain three pos2/color4 vertices"); + auto vertex_buffer = + CreateHostBuffer(test.name, vertices.size() * sizeof(u32), + vk::BufferUsageFlagBits::eVertexBuffer, vertices); + + auto make_module = [&](const std::vector &spirv) { + vk::ShaderModuleCreateInfo module_info{}; + module_info.sType = vk::StructureType::eShaderModuleCreateInfo; + module_info.codeSize = spirv.size() * sizeof(u32); + module_info.pCode = spirv.data(); + vk::ShaderModule module = nullptr; + RequireVk(test.name, "graphics", + m_device.createShaderModule(&module_info, nullptr, &module), + "vkCreateShaderModule"); + return module; + }; + vk::ShaderModule vertex_module = make_module(vertex_spirv); + vk::ShaderModule fragment_module = make_module(fragment.spirv); + + const auto &fragment_bind = fragment.program.bindings; + vk::PushConstantRange push_constant_range{}; + if (fragment_bind.push_constant_size > 0) { + Require(test.name, "graphics", + test.push_constants.size() * sizeof(u32) == + fragment_bind.push_constant_size, + "fragment push constant data size does not match reflection"); + push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment; + push_constant_range.offset = fragment_bind.push_constant_offset; + push_constant_range.size = fragment_bind.push_constant_size; + } + + vk::PipelineLayoutCreateInfo pipeline_layout_info{}; + pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; + pipeline_layout_info.pushConstantRangeCount = + push_constant_range.size != 0 ? 1u : 0u; + pipeline_layout_info.pPushConstantRanges = + push_constant_range.size != 0 ? &push_constant_range : nullptr; + vk::PipelineLayout pipeline_layout = nullptr; + RequireVk(test.name, "graphics", + m_device.createPipelineLayout(&pipeline_layout_info, nullptr, + &pipeline_layout), + "vkCreatePipelineLayout"); + + vk::PipelineShaderStageCreateInfo stages[2] = {}; + stages[0].sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stages[0].stage = vk::ShaderStageFlagBits::eVertex; + stages[0].module = vertex_module; + stages[0].pName = "main"; + stages[1].sType = vk::StructureType::ePipelineShaderStageCreateInfo; + stages[1].stage = vk::ShaderStageFlagBits::eFragment; + stages[1].module = fragment_module; + stages[1].pName = "main"; + + vk::VertexInputBindingDescription vertex_binding{}; + vertex_binding.binding = 0; + vertex_binding.stride = 6u * sizeof(float); + vertex_binding.inputRate = vk::VertexInputRate::eVertex; + vk::VertexInputAttributeDescription attributes[2] = {}; + attributes[0].location = 0; + attributes[0].binding = 0; + attributes[0].format = vk::Format::eR32G32Sfloat; + attributes[0].offset = 0; + attributes[1].location = 1; + attributes[1].binding = 0; + attributes[1].format = vk::Format::eR32G32B32A32Sfloat; + attributes[1].offset = 2u * sizeof(float); + vk::PipelineVertexInputStateCreateInfo vertex_input{}; + vertex_input.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo; + vertex_input.vertexBindingDescriptionCount = 1; + vertex_input.pVertexBindingDescriptions = &vertex_binding; + vertex_input.vertexAttributeDescriptionCount = 2; + vertex_input.pVertexAttributeDescriptions = attributes; + + vk::PipelineInputAssemblyStateCreateInfo input_assembly{}; + input_assembly.sType = + vk::StructureType::ePipelineInputAssemblyStateCreateInfo; + input_assembly.topology = vk::PrimitiveTopology::eTriangleList; + + vk::Viewport viewport{}; + viewport.x = 0.0f; + viewport.y = 0.0f; + viewport.width = 1.0f; + viewport.height = 1.0f; + viewport.minDepth = 0.0f; + viewport.maxDepth = 1.0f; + vk::Rect2D scissor{}; + scissor.extent.width = 1; + scissor.extent.height = 1; + vk::PipelineViewportStateCreateInfo viewport_state{}; + viewport_state.sType = vk::StructureType::ePipelineViewportStateCreateInfo; + viewport_state.viewportCount = 1; + viewport_state.pViewports = &viewport; + viewport_state.scissorCount = 1; + viewport_state.pScissors = &scissor; + + vk::PipelineRasterizationStateCreateInfo raster{}; + raster.sType = vk::StructureType::ePipelineRasterizationStateCreateInfo; + raster.polygonMode = vk::PolygonMode::eFill; + raster.cullMode = vk::CullModeFlagBits::eNone; + raster.frontFace = vk::FrontFace::eCounterClockwise; + raster.lineWidth = 1.0f; + + vk::PipelineMultisampleStateCreateInfo multisample{}; + multisample.sType = vk::StructureType::ePipelineMultisampleStateCreateInfo; + multisample.rasterizationSamples = vk::SampleCountFlagBits::e1; + + vk::PipelineColorBlendAttachmentState color_attachment{}; + color_attachment.colorWriteMask = + vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | + vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA; + vk::PipelineColorBlendStateCreateInfo color_blend{}; + color_blend.sType = vk::StructureType::ePipelineColorBlendStateCreateInfo; + color_blend.attachmentCount = 1; + color_blend.pAttachments = &color_attachment; + + vk::GraphicsPipelineCreateInfo pipeline_info{}; + const vk::Format color_format = vk::Format::eR32G32B32A32Sfloat; + vk::PipelineRenderingCreateInfo rendering_pipeline{}; + rendering_pipeline.sType = vk::StructureType::ePipelineRenderingCreateInfo; + rendering_pipeline.colorAttachmentCount = 1; + rendering_pipeline.pColorAttachmentFormats = &color_format; + pipeline_info.sType = vk::StructureType::eGraphicsPipelineCreateInfo; + pipeline_info.pNext = &rendering_pipeline; + pipeline_info.stageCount = 2; + pipeline_info.pStages = stages; + pipeline_info.pVertexInputState = &vertex_input; + pipeline_info.pInputAssemblyState = &input_assembly; + pipeline_info.pViewportState = &viewport_state; + pipeline_info.pRasterizationState = &raster; + pipeline_info.pMultisampleState = &multisample; + pipeline_info.pColorBlendState = &color_blend; + pipeline_info.layout = pipeline_layout; + vk::Pipeline pipeline = nullptr; + RequireVk(test.name, "graphics", + m_device.createGraphicsPipelines(nullptr, 1, &pipeline_info, + nullptr, &pipeline), + "vkCreateGraphicsPipelines"); + + vk::CommandBuffer cmd = BeginCommands(test.name, "graphics"); + vk::RenderingAttachmentInfo color{}; + color.sType = vk::StructureType::eRenderingAttachmentInfo; + color.imageView = target.view; + color.imageLayout = vk::ImageLayout::eGeneral; + color.loadOp = vk::AttachmentLoadOp::eClear; + color.storeOp = vk::AttachmentStoreOp::eStore; + vk::RenderingInfo rendering{}; + rendering.sType = vk::StructureType::eRenderingInfo; + rendering.renderArea.extent = {1, 1}; + rendering.layerCount = 1; + rendering.colorAttachmentCount = 1; + rendering.pColorAttachments = &color; + cmd.beginRendering(rendering); + cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); + if (push_constant_range.size != 0) { + cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, + fragment_bind.push_constant_offset, + fragment_bind.push_constant_size, + test.push_constants.data()); + } + vk::DeviceSize offset = 0; + cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset); + cmd.draw(3, 1, 0, 0); + cmd.endRendering(); + EndSubmitAndFree(test.name, "graphics", cmd); + target.layout = vk::ImageLayout::eGeneral; + + auto pixel = ReadImage(test.name, &target); + pixel.resize(4); + + m_device.destroyPipeline(pipeline, nullptr); + m_device.destroyPipelineLayout(pipeline_layout, nullptr); + m_device.destroyShaderModule(fragment_module, nullptr); + m_device.destroyShaderModule(vertex_module, nullptr); + DestroyBuffer(&vertex_buffer); + DestroyImage(&target); + return pixel; + } + + void CheckGpuTilerCpuParity() { + constexpr const char *name = "GpuTilerCpuParity"; + EnsureRuntimeContext(); + + CommandScheduler scheduler(Renderer(), m_runtime_context); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + StreamBuffer parameters(m_runtime_context, scheduler, MemoryUsage::Stream, + 1u << 20); + TileManager tile_manager(m_runtime_context, scheduler, parameters); + constexpr uint64_t conversion_limit = 128ull << 20; + Require(name, "D16 conversion chunk boundaries", + TileManagerTestAccess::ConversionRows(0, 32ull << 10, 32ull << 10, + 4096, 256, conversion_limit, + 65535) == 4096 && + TileManagerTestAccess::ConversionRows( + 0, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit, + 65535) == 4096 && + TileManagerTestAccess::ConversionRows( + 257, 32ull << 10, 32ull << 10, 4097, 256, conversion_limit, + 65535) == 4095, + "D16 conversion planner exceeds descriptor or dispatch limits"); + + const auto align_dword = [](uint64_t size) { + return (size + sizeof(u32) - 1u) & ~(uint64_t{sizeof(u32) - 1u}); + }; + const auto to_dwords = [&](const std::vector &bytes, + uint64_t padded_size) { + std::vector words(static_cast(padded_size / sizeof(u32)), 0); + std::memcpy(words.data(), bytes.data(), bytes.size()); + return words; + }; + const auto read_bytes = [&](const Buffer &buffer, uint64_t size) { + const auto words = ReadBuffer( + name, buffer, static_cast(align_dword(size) / sizeof(u32))); + std::vector bytes(static_cast(size)); + std::memcpy(bytes.data(), words.data(), bytes.size()); + return bytes; + }; + const auto host_barrier = [&](vk::Buffer buffer, uint64_t size, + vk::PipelineStageFlags source_stage, + vk::AccessFlags source_access) { + vk::BufferMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eBufferMemoryBarrier; + barrier.srcAccessMask = source_access; + barrier.dstAccessMask = vk::AccessFlagBits::eHostRead; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = 0; + barrier.size = size; + scheduler.Current().Handle().pipelineBarrier( + source_stage, vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, + &barrier, 0, nullptr); + }; + const auto gpu_detile = + [&](const std::vector &tiled, std::vector *linear, + uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos) { + const uint64_t padded_tiled = align_dword(tiled_capacity); + const uint64_t padded_linear = align_dword(linear_capacity); + auto input = CreateHostBuffer(name, padded_tiled, AllFlags, + to_dwords(tiled, padded_tiled)); + auto output = CreateHostBuffer( + name, padded_linear, AllFlags, + std::vector(static_cast(padded_linear / sizeof(u32)), + 0xabababab)); + const auto result = tile_manager.Detile(input.buffer, 0, padded_tiled, + padded_linear, infos); + const vk::BufferCopy copy{result.offset, 0, padded_linear}; + scheduler.Current().Handle().copyBuffer(result.buffer, output.buffer, + 1, ©); + host_barrier(output.buffer, padded_linear, + vk::PipelineStageFlagBits::eTransfer, + vk::AccessFlagBits::eTransferWrite); + scheduler.Finish(); + *linear = read_bytes(output, linear_capacity); + DestroyBuffer(&output); + DestroyBuffer(&input); + }; + const auto gpu_tile = [&](const std::vector &linear, + std::vector *tiled, + uint64_t tiled_capacity, uint64_t linear_capacity, + std::span infos) { + const uint64_t padded_tiled = align_dword(tiled_capacity); + const uint64_t padded_linear = align_dword(linear_capacity); + auto input = CreateHostBuffer(name, padded_linear, AllFlags, + to_dwords(linear, padded_linear)); + auto output = CreateHostBuffer( + name, padded_tiled, AllFlags, + std::vector(static_cast(padded_tiled / sizeof(u32)), + 0xabababab)); + tile_manager.Tile(input.buffer, 0, padded_linear, output.buffer, 0, + padded_tiled, infos); + host_barrier(output.buffer, padded_tiled, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eShaderWrite); + scheduler.Finish(); + *tiled = read_bytes(output, tiled_capacity); + DestroyBuffer(&output); + DestroyBuffer(&input); + }; + + for (const bool d32 : std::array{true, false}) { + constexpr uint32_t width = 5; + constexpr uint32_t height = 3; + constexpr uint32_t pitch = 8; + constexpr uint32_t layers = 2; + constexpr uint64_t d16_row = pitch * sizeof(uint16_t); + constexpr uint64_t d32_row = pitch * sizeof(uint32_t); + constexpr uint64_t d16_slice = d16_row * height; + constexpr uint64_t d32_slice = d32_row * height; + constexpr uint64_t d16_size = d16_slice * layers; + constexpr uint64_t d32_size = d32_slice * layers; + StreamBuffer conversion_upload(m_runtime_context, scheduler, + MemoryUsage::Upload, 1u << 20); + auto [prefix, prefix_offset] = conversion_upload.Map(4096, 256); + Require(name, "D16 conversion prefix", + prefix != nullptr && prefix_offset == 0, + "failed to advance the D16 conversion upload ring"); + std::memset(prefix, 0x7b, 4096); + conversion_upload.Commit(); + auto [source_bytes, source_offset] = conversion_upload.Map(d16_size, 4); + Require(name, "D16 conversion source", + source_bytes != nullptr && source_offset != 0, + "failed to reserve a nonzero D16 conversion source"); + std::memset(source_bytes, 0x3c, d16_size); + for (uint32_t layer = 0; layer < layers; layer++) { + for (uint32_t y = 0; y < height; y++) { + auto *row = reinterpret_cast( + source_bytes + layer * d16_slice + y * d16_row); + for (uint32_t x = 0; x < width; x++) { + row[x] = static_cast( + 0x1111u * (1u + x + y * width + layer * width * height)); + } + } + } + conversion_upload.Commit(); + auto promoted = CreateHostBuffer( + name, d32_size, AllFlags, + std::vector(static_cast(d32_size / sizeof(u32)), 0)); + auto round_trip = CreateHostBuffer( + name, d16_size, AllFlags, + std::vector(static_cast(d16_size / sizeof(u32)), + 0xa5a5a5a5u)); + const TileManager::D16Layout promote_layout{ + .width = width, + .height = height, + .layers = layers, + .source_row_stride = d16_row, + .target_row_stride = d32_row, + .source_slice_stride = d16_slice, + .target_slice_stride = d32_slice, + }; + tile_manager.ConvertD16({conversion_upload.Handle(), source_offset, + conversion_upload.Size() - source_offset}, + {promoted.buffer, 0, d32_size}, + TileManager::D16Direction::Promote, d32, + promote_layout); + tile_manager.ConvertD16({promoted.buffer, 0, d32_size}, + {round_trip.buffer, 0, d16_size}, + TileManager::D16Direction::Demote, d32, + {.width = width, + .height = height, + .layers = layers, + .source_row_stride = d32_row, + .target_row_stride = d16_row, + .source_slice_stride = d32_slice, + .target_slice_stride = d16_slice}); + host_barrier(round_trip.buffer, d16_size, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eShaderWrite); + scheduler.Finish(); + const auto result = read_bytes(round_trip, d16_size); + bool conversion_matches = true; + for (uint32_t layer = 0; layer < layers; layer++) { + for (uint32_t y = 0; y < height; y++) { + for (uint32_t x = 0; x < pitch; x++) { + const auto offset = + layer * d16_slice + y * d16_row + x * sizeof(uint16_t); + conversion_matches &= + x < width + ? std::memcmp(result.data() + offset, source_bytes + offset, + sizeof(uint16_t)) == 0 + : result[offset] == 0xa5 && result[offset + 1] == 0xa5; + } + } + } + Require(name, + d32 ? "D32 conversion round trip" : "D24 conversion round trip", + conversion_matches, + "D16 conversion lost rows, layers, active values, or padding"); + DestroyBuffer(&round_trip); + DestroyBuffer(&promoted); + } + + auto fill = [](std::vector *bytes, u32 salt) { + for (size_t i = 0; i < bytes->size(); i++) { + uint64_t value = + i + static_cast(salt) * 0x9e3779b97f4a7c15ull; + value = (value ^ (value >> 30u)) * 0xbf58476d1ce4e5b9ull; + value = (value ^ (value >> 27u)) * 0x94d049bb133111ebull; + (*bytes)[i] = static_cast((value ^ (value >> 31u)) >> 56u); + } + }; + auto compare = [&](const char *stage, const std::vector &expected, + const std::vector &actual) { + if (expected != actual) { + const auto mismatch = static_cast( + std::mismatch(expected.begin(), expected.end(), actual.begin()) + .first - + expected.begin()); + std::ostringstream out; + out << "first mismatch at " << mismatch << " of " << expected.size(); + Fail(name, stage, out.str()); + } + }; + auto convert_reference = [&](bool to_tiled, std::vector *dst, + const std::vector &src, + const GpuTileInfo &info) { + TileBlockLayout block{}; + Require(name, "reference layout", + TileGetBlockLayout(info.family, info.bytes_per_element, block), + "CPU reference rejected GPU tile info"); + const u32 tiled_width = + info.tiled_width != 0 ? info.tiled_width : info.pitch; + const u32 tiled_height = + info.tiled_height != 0 ? info.tiled_height : info.height; + const uint64_t columns = + (tiled_width + block.block_width - 1u) / block.block_width; + const uint64_t rows = + (tiled_height + block.block_height - 1u) / block.block_height; + const uint64_t slice = info.linear_slice_stride != 0 + ? info.linear_slice_stride + : static_cast(info.pitch) * + info.height * info.bytes_per_element; + for (u32 z = 0; z < info.depth; ++z) { + for (u32 y = 0; y < info.height; ++y) { + for (u32 x = 0; x < info.width; ++x) { + const u32 bx = info.tail ? 0 : x / block.block_width; + const u32 by = info.tail ? 0 : y / block.block_height; + const u32 bz = info.tail ? 0 : z / block.block_depth; + const u32 lx = info.tail ? x + info.tail_x : x % block.block_width; + const u32 ly = info.tail ? y + info.tail_y : y % block.block_height; + const u32 lz = z % block.block_depth; + u32 local = 0, block_xor = 0; + Require(name, "reference offset", + TileGetBlockOffset(block, lx, ly, lz, local) && + TileGetBlockXor(block, bx, by, info.surface_z + bz, + block_xor), + "CPU reference address lookup failed"); + const uint64_t block_index = + static_cast(bz) * columns * rows + by * columns + bx; + const uint64_t tiled = info.tiled_offset + + block_index * block.block_size + + (local ^ block_xor); + const uint64_t linear = + info.linear_offset + static_cast(z) * slice + + static_cast(y) * info.pitch * info.bytes_per_element + + static_cast(x) * info.bytes_per_element; + const uint64_t dst_offset = to_tiled ? tiled : linear; + const uint64_t src_offset = to_tiled ? linear : tiled; + Require(name, "reference range", + dst_offset + info.bytes_per_element <= dst->size() && + src_offset + info.bytes_per_element <= src.size(), + "CPU reference address escaped storage"); + std::memcpy(dst->data() + dst_offset, src.data() + src_offset, + info.bytes_per_element); + } + } + } + }; + struct FamilyCase { + TileBlockFamily family; + u32 max_bpe; + }; + constexpr FamilyCase families[] = { + {TileBlockFamily::Standard256B, 16}, + {TileBlockFamily::Standard4KB, 16}, + {TileBlockFamily::Standard4KB3D, 16}, + {TileBlockFamily::Standard64KB, 16}, + {TileBlockFamily::Standard64KB3D, 16}, + {TileBlockFamily::Prt64KB, 16}, + {TileBlockFamily::Prt64KB3D, 16}, + {TileBlockFamily::RenderTarget64KB, 16}, + {TileBlockFamily::Depth64KB, 8}, + }; + + { + TileBlockLayout standard{}, prt{}, standard_3d{}, prt_3d{}, color{}, + depth{}; + u32 standard_offset = 0, prt_offset = 0, standard_3d_offset = 0, + prt_3d_offset = 0, color_z = 0, depth_z = 0; + bool fixed_addresses = + TileGetBlockLayout(TileBlockFamily::Standard64KB, 4, standard) && + TileGetBlockLayout(TileBlockFamily::Prt64KB, 4, prt) && + TileGetBlockLayout(TileBlockFamily::Standard64KB3D, 4, standard_3d) && + TileGetBlockLayout(TileBlockFamily::Prt64KB3D, 4, prt_3d) && + TileGetBlockLayout(TileBlockFamily::RenderTarget64KB, 4, color) && + TileGetBlockLayout(TileBlockFamily::Depth64KB, 4, depth) && + TileGetBlockOffset(standard, 64, 0, 0, standard_offset) && + TileGetBlockOffset(prt, 64, 0, 0, prt_offset) && + TileGetBlockOffset(standard_3d, 16, 0, 0, standard_3d_offset) && + TileGetBlockOffset(prt_3d, 16, 0, 0, prt_3d_offset) && + TileGetBlockXor(color, 0, 0, 1, color_z) && + TileGetBlockXor(depth, 0, 0, 15, depth_z); + Require(name, "fixed address vectors", + fixed_addresses && standard_offset == 0x8000 && + prt_offset == 0x8100 && standard_3d_offset == 0x8000 && + prt_3d_offset == 0x8400 && standard_3d.block_width == 32 && + standard_3d.block_height == 32 && + standard_3d.block_depth == 16 && color_z == 0x800 && + depth_z == 0xf00, + "a fixed block address changed"); + + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr u32 levels = 7; + TileSizeAlign total{}; + TileSizeOffset mip[levels]{}; + TilePaddedSize padded[levels]{}; + TileGetTextureSize(format, 65, 33, levels, + Prospero::TileMode::kStandard256B, &total, mip, + padded); + constexpr u32 offsets[levels] = {0x1a00, 0xb00, 0x500, 0x300, + 0x200, 0x100, 0}; + constexpr u32 sizes[levels] = {0x2d00, 0xf00, 0x600, 0x200, + 0x100, 0x100, 0x100}; + bool layout_matches = total.size == 0x4700 && total.align == 0x100 && + padded[0].width == 72 && padded[0].height == 40; + for (u32 level = 0; level < levels; ++level) { + layout_matches &= mip[level].offset == offsets[level] && + mip[level].size == sizes[level]; + } + Require(name, "fixed mip vector", layout_matches, + "the reverse-packed small-block mip layout changed"); + } + + u32 case_index = 0; + auto check_round_trip = [&](const char *stage, uint64_t tiled_size, + std::span infos) { + uint64_t linear_size = 0; + for (const auto &info : infos) { + linear_size = + std::max(linear_size, info.linear_offset + info.linear_size); + } + std::vector tiled(tiled_size); + std::vector cpu(linear_size, 0); + std::vector gpu(linear_size, 0xab); + fill(&tiled, ++case_index); + for (const auto &info : infos) { + convert_reference(false, &cpu, tiled, info); + } + gpu_detile(tiled, &gpu, tiled_size, linear_size, infos); + compare((std::string(stage) + " detile bytes").c_str(), cpu, gpu); + + std::vector linear(linear_size); + std::vector cpu_tiled(tiled_size, 0xab); + std::vector gpu_tiled(tiled_size, 0xab); + fill(&linear, 0x280u + case_index); + for (const auto &info : infos) { + convert_reference(true, &cpu_tiled, linear, info); + } + gpu_tile(linear, &gpu_tiled, tiled_size, linear_size, infos); + compare((std::string(stage) + " tile bytes").c_str(), cpu_tiled, + gpu_tiled); + }; + for (const auto family : families) { + for (u32 bpe = 1; bpe <= family.max_bpe; bpe <<= 1u) { + TileBlockLayout block{}; + Require(name, "block layout", + TileGetBlockLayout(family.family, bpe, block), + "admitted family/BPE has no block layout"); + const bool volume = block.block_depth > 1; + const u32 width = + block.block_width * 3u + std::min(block.block_width, 3u); + const u32 height = + block.block_height * 3u + std::min(block.block_height, 3u); + const u32 depth = volume ? block.block_depth + 1u : 1u; + const u32 pitch = block.block_width * 4u; + const uint64_t block_columns = + (pitch + block.block_width - 1u) / block.block_width; + const uint64_t block_rows = + (height + block.block_height - 1u) / block.block_height; + const uint64_t block_slices = + (depth + block.block_depth - 1u) / block.block_depth; + const uint64_t storage_size = + block_columns * block_rows * block_slices * block.block_size; + const uint64_t slice_stride = + static_cast(pitch) * height * bpe; + + std::vector tiled(storage_size); + std::vector cpu(storage_size, 0); + std::vector gpu(storage_size, 0xab); + fill(&tiled, ++case_index); + + GpuTileInfo info{}; + info.family = block.family; + info.bytes_per_element = block.bytes_per_element; + info.linear_size = storage_size; + info.tiled_size = storage_size; + info.linear_slice_stride = volume ? slice_stride : 0; + info.width = width; + info.height = height; + info.depth = depth; + info.pitch = pitch; + info.surface_z = family.family == TileBlockFamily::RenderTarget64KB || + family.family == TileBlockFamily::Depth64KB + ? 3 + : 0; + convert_reference(false, &cpu, tiled, info); + gpu_detile(tiled, &gpu, storage_size, storage_size, + std::span(&info, 1)); + const auto family_label = [&](const char *operation) { + std::ostringstream out; + out << operation << " family=" << static_cast(family.family) + << " bpe=" << bpe; + return out.str(); + }; + compare(family_label("detile bytes").c_str(), cpu, gpu); + + { + std::vector linear(storage_size); + std::vector cpu_tiled(storage_size, 0xab); + std::vector gpu_tiled(storage_size, 0xab); + fill(&linear, 0x80u + case_index); + convert_reference(true, &cpu_tiled, linear, info); + gpu_tile(linear, &gpu_tiled, storage_size, storage_size, + std::span(&info, 1)); + compare(family_label("tile bytes").c_str(), cpu_tiled, gpu_tiled); + } + } + } + + // Exercise the real texture-layout/info-building seam for every format + // admitted by each standard tile mode. Formats sharing a byte/block width + // intentionally share a shader, but this loop still validates their + // texel-to-element conversion (notably every BCn format). + struct StandardMode { + Prospero::TileMode tile; + TileBlockFamily family; + }; + constexpr StandardMode standard_modes[] = { + {Prospero::TileMode::kStandard256B, TileBlockFamily::Standard256B}, + {Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB}, + {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB}, + {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB}, + }; + for (const auto format : + {Prospero::BufferFormat::k8Srgb, Prospero::BufferFormat::k8_8Srgb, + Prospero::BufferFormat::k9_9_9_5Float}) { + for (const auto tile : + {Prospero::TileMode::kDepth, Prospero::TileMode::kRenderTarget}) { + TileTextureBlockLayout texture{}; + Require(name, "RT format policy", + !TileGetTextureBlockLayout(format, tile, false, texture), + "non-render-target format admitted by an RT/depth tile family"); + } + } + { + TileSurfaceLayout invalid{}; + const TileSurfaceDescription description{ + Prospero::BufferFormat::k32Float, + Prospero::TileMode::kStandard256B, + TileSurfaceDimension::Dim3D, + 16, + 16, + 16, + 1, + 1, + }; + Require(name, "3D tile policy", + !TileGetTiledTextureLayout(description, invalid), + "Standard256B volume silently fell back to a 2D array layout"); + } + { + constexpr auto format = Prospero::BufferFormat::kFmask8_S4_F4; + constexpr auto tile = Prospero::TileMode::kStandard64KB; + TileSizeAlign total{}; + TileGetTextureSize(format, 128, 128, 1, tile, &total, nullptr, nullptr); + const auto layout = TextureCalcUploadLayout( + format, 128, 128, 1, 1, tile, total.size, false, false, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + Require(name, "FMASK policy", + !TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos), + "FMASK entered the texel tiler through a non-depth family"); + } + u32 format_cases = 0; + for (u32 raw_format = 1; + raw_format <= static_cast(Prospero::BufferFormat::kBc7Srgb); + ++raw_format) { + const auto format = static_cast(raw_format); + // FMASK is synthesized as identity metadata by TextureUploadFmask; it is + // deliberately not a texel surface and must never enter the GPU tiler. + if (Prospero::IsFmaskTextureFormat(format)) { + continue; + } + for (const auto &mode : standard_modes) { + TileTextureBlockLayout texture{}; + if (!TileGetTextureBlockLayout(format, mode.tile, false, texture)) { + continue; + } + const u32 bpe = texture.block.bytes_per_element; + Require(name, "format block", texture.block.family == mode.family, + "CPU-supported format selected the wrong block family"); + + constexpr u32 width = 67; + constexpr u32 height = 51; + const u32 pitch = TileGetTexturePitch(format, width, mode.tile); + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, 1, mode.tile, &total, nullptr, + nullptr); + Require(name, "format size", total.size != 0, + "supported format has an empty layout"); + + const auto layout = + TextureCalcUploadLayout(format, width, height, 1, 1, mode.tile, + total.size, false, false, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + if (!TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos)) { + std::ostringstream out; + out << "format=" << raw_format + << " tile=" << static_cast(mode.tile) + << " size=" << total.size << " pitch=" << pitch; + Fail(name, "format infos", out.str()); + } + Require(name, "format family", + infos.size() == 1 && infos[0].family == mode.family && + infos[0].bytes_per_element == bpe, + "texture info selected the wrong shader family"); + + std::vector tiled(total.size); + std::vector cpu(total.size, 0); + std::vector gpu(total.size, 0xab); + fill(&tiled, ++case_index); + convert_reference(false, &cpu, tiled, infos[0]); + gpu_detile(tiled, &gpu, total.size, total.size, infos); + compare("format bytes", cpu, gpu); + ++format_cases; + } + } + Require(name, "format coverage", format_cases != 0, + "no CPU-supported standard formats were tested"); + + { + constexpr auto format = Prospero::BufferFormat::k32_32_32Float; + TileTextureElementLayout element{}; + Require(name, "RGB32 texture policy", + !TileGetTextureElementLayout(format, element), + "12-byte texel layouts must be rejected"); + } + + { + constexpr auto format = Prospero::BufferFormat::kBc1UNorm; + constexpr auto tile = Prospero::TileMode::kStandard64KB; + constexpr u32 width = 256, height = 256, levels = 9; + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, levels, tile, &total, nullptr, + nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, tile, + total.size, false, false, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + const bool built = + TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); + uint64_t linear_size = 0; + for (const auto &info : infos) { + linear_size = + std::max(linear_size, info.linear_offset + info.linear_size); + } + Require(name, "BC1 mip-tail capacities", + built && total.size == 0x10000 && + layout.surface.first_tail_level == 0 && + linear_size == 0x15560 && linear_size > total.size, + "BC1 mip tail conflated tiled and linear capacities"); + check_round_trip("BC1 mip tail", total.size, infos); + } + + { + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr auto tile = Prospero::TileMode::kRenderTarget; + constexpr u32 width = 129, height = 65, layers = 3; + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, layers, 1, tile, false, + total); + const auto layout = + TextureCalcUploadLayout(format, width, height, 1, layers, tile, + total.size, true, false, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + const bool built = + TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos); + Require(name, "array infos", + built && infos.size() == layers && infos[0].surface_z == 0 && + infos[1].surface_z == 1 && infos[2].surface_z == 2, + "array slices lost their absolute surface Z"); + check_round_trip("array", total.size, infos); + } + + for (const auto &mode : standard_modes) { + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr u32 levels = 2; + TileBlockLayout block{}; + Require(name, "odd mip block", TileGetBlockLayout(mode.family, 4, block), + "odd multi-mip format has no block layout"); + const u32 width = block.block_width * 2u + 1u; + const u32 height = block.block_height * 2u + 1u; + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, levels, mode.tile, &total, + nullptr, nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile, + total.size, false, false, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + Require(name, "odd mip infos", + TextureBuildGpuTileInfos(total.size, regions, layout, levels, + infos) && + infos.size() == 2 && infos[1].tiled_width >= infos[1].pitch && + infos[1].tiled_height >= infos[1].height && + (mode.family == TileBlockFamily::Standard256B || + infos[1].tiled_height > infos[1].height), + "odd multi-mip physical stride collapsed to the active linear " + "extent"); + check_round_trip("odd multi-mip", total.size, infos); + } + + struct TailMode { + Prospero::TileMode tile; + TileBlockFamily family; + }; + constexpr TailMode tail_modes[] = { + {Prospero::TileMode::kStandard4KB, TileBlockFamily::Standard4KB}, + {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB}, + {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB}, + {Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB}, + {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB}, + }; + for (const auto &mode : tail_modes) { + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr u32 levels = 7; + TileBlockLayout block{}; + Require(name, "2D tail block", TileGetBlockLayout(mode.family, 4, block), + "2D tail mode has no block layout"); + const u32 width = block.block_width * 2u; + const u32 height = block.block_height * 2u; + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, levels, mode.tile, &total, + nullptr, nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, mode.tile, + total.size, true, false, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + Require(name, "2D mip tail seam", + layout.surface.first_tail_level == 2 && + TextureBuildGpuTileInfos(total.size, regions, layout, levels, + infos) && + infos.size() == levels && !infos[0].tail && !infos[1].tail && + std::all_of(infos.begin() + 2, infos.end(), + [](const auto &info) { return info.tail; }), + "2D mip chain lost its linear/tiled tail boundary"); + check_round_trip("2D mip tail seam", total.size, infos); + } + + { + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr auto tile = Prospero::TileMode::kStandard4KB; + constexpr u32 width = 65; + constexpr u32 height = 129; + constexpr u32 depth = 17; + constexpr u32 levels = 2; + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, + total); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, depth, tile, + total.size, false, true, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + const bool built = + TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); + Require(name, "3D mip infos", + regions.size() == depth + (depth >> 1u) && built && + infos.size() == 4 && infos[0].depth == 8 && + infos[1].depth == 8 && infos[2].depth == 1 && + infos[3].depth == 8 && infos[0].tiled_offset == 0x19000 && + infos[1].tiled_offset == 0x83000 && + infos[2].tiled_offset == 0xed000 && + infos[3].tiled_offset == 0 && infos[3].pitch == 32 && + infos[3].height == 64 && infos[3].tiled_width == 40 && + infos[3].tiled_height == 80, + "Standard4KB3D mip depth, packing, or physical stride changed"); + check_round_trip("3D mip", total.size, infos); + } + + { + constexpr auto format = Prospero::BufferFormat::kBc1UNorm; + constexpr auto tile = Prospero::TileMode::kStandard4KB; + constexpr u32 width = 65; + constexpr u32 height = 129; + constexpr u32 depth = 17; + constexpr u32 levels = 2; + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, + total); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, depth, tile, + total.size, false, true, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + Require(name, "3D BC mip infos", + TextureBuildGpuTileInfos(total.size, regions, layout, levels, + infos) && + infos.size() == 4 && infos[3].pitch == 8 && + infos[3].height == 16 && infos[3].tiled_width == 16 && + infos[3].tiled_height == 24, + "block-compressed 3D mip lost its physical row or slice stride"); + check_round_trip("3D BC mip", total.size, infos); + } + + { + constexpr auto format = Prospero::BufferFormat::kBc3UNorm; + constexpr auto tile = Prospero::TileMode::kLinear; + constexpr u32 width = 8, height = 8, levels = 4; + TileSizeAlign total{}; + TileGetTextureSize(format, width, height, levels, tile, &total, nullptr, + nullptr); + const auto layout = + TextureCalcUploadLayout(format, width, height, levels, 1, tile, + total.size, false, false, name); + const auto regions = TextureBuildImageCopies(layout); + Require(name, "linear BC native regions", + regions.size() == levels && + std::all_of(regions.begin(), regions.end(), + [](const auto ®ion) { + return region.bufferRowLength >= + region.imageExtent.width && + region.bufferImageHeight >= + region.imageExtent.height && + region.bufferRowLength % 4 == 0 && + region.bufferImageHeight % 4 == 0; + }), + "linear BC mip tails emitted invalid Vulkan copy strides"); + } + + struct VolumeModeCase { + Prospero::TileMode tile; + TileBlockFamily family; + Prospero::BufferFormat format; + }; + constexpr VolumeModeCase volume_modes[] = { + {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D, + Prospero::BufferFormat::k32Float}, + {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D, + Prospero::BufferFormat::k32Float}, + {Prospero::TileMode::kRenderTarget, TileBlockFamily::RenderTarget64KB, + Prospero::BufferFormat::k32Float}, + {Prospero::TileMode::kDepth, TileBlockFamily::Depth64KB, + Prospero::BufferFormat::k32Float}, + {Prospero::TileMode::kStandard64KB, TileBlockFamily::Standard64KB3D, + Prospero::BufferFormat::kBc1UNorm}, + {Prospero::TileMode::kPrt, TileBlockFamily::Prt64KB3D, + Prospero::BufferFormat::kBc3UNorm}, + }; + for (const auto test : volume_modes) { + constexpr u32 width = 65, height = 33, depth = 37, levels = 6; + TileSizeAlign total{}; + TileGetTextureTotalSize(test.format, width, height, depth, levels, + test.tile, true, total); + const auto layout = + TextureCalcUploadLayout(test.format, width, height, levels, depth, + test.tile, total.size, true, true, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + const bool built = + TextureBuildGpuTileInfos(total.size, regions, layout, levels, infos); + const bool uses_z = test.family == TileBlockFamily::RenderTarget64KB || + test.family == TileBlockFamily::Depth64KB; + Require(name, "volume family infos", + built && !infos.empty() && + std::all_of(infos.begin(), infos.end(), + [&](const auto &info) { + return info.family == test.family; + }) && + (!uses_z || std::any_of(infos.begin(), infos.end(), + [](const auto &info) { + return info.surface_z != 0; + })), + "volume mode selected the wrong family or lost its surface Z"); + check_round_trip("volume family", total.size, infos); + } + + struct VolumeTailCase { + Prospero::BufferFormat format; + u32 bytes; + }; + constexpr VolumeTailCase volume_tails[] = { + {Prospero::BufferFormat::k8UNorm, 1}, + {Prospero::BufferFormat::k16UNorm, 2}, + {Prospero::BufferFormat::k32Float, 4}, + {Prospero::BufferFormat::k16_16_16_16Float, 8}, + {Prospero::BufferFormat::k32_32_32_32Float, 16}, + {Prospero::BufferFormat::kBc1UNorm, 8}, + {Prospero::BufferFormat::kBc3UNorm, 16}, + }; + constexpr u32 volume_tail_xy[5][5][2] = { + {{0, 8}, {8, 4}, {8, 0}, {0, 4}, {0, 0}}, + {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{0, 8}, {4, 4}, {4, 0}, {0, 4}, {0, 0}}, + {{0, 4}, {4, 2}, {4, 0}, {0, 2}, {0, 0}}, + {{0, 4}, {2, 2}, {2, 0}, {0, 2}, {0, 0}}, + }; + for (const auto &tail : volume_tails) { + constexpr auto tile = Prospero::TileMode::kStandard4KB; + constexpr u32 levels = 5; + TileBlockLayout block{}; + Require( + name, "3D tail block", + TileGetBlockLayout(TileBlockFamily::Standard4KB3D, tail.bytes, block), + "Standard4KB3D tail format has no block layout"); + const bool compressed = + Prospero::BlockCompressedBytesPerBlock(tail.format) != 0; + const u32 width = block.block_width * (compressed ? 4u : 1u); + const u32 height = block.block_height / 2u * (compressed ? 4u : 1u); + const u32 depth = block.block_depth + 1u; + TileSizeAlign total{}; + TileGetTextureTotalSize(tail.format, width, height, depth, levels, tile, + true, total); + const auto layout = + TextureCalcUploadLayout(tail.format, width, height, levels, depth, + tile, total.size, false, true, name); + const auto regions = TextureBuildImageCopies(layout); + std::vector infos; + bool valid = total.size == 8192 && + regions.size() == depth + std::max(depth >> 1u, 1u) + + std::max(depth >> 2u, 1u) + + std::max(depth >> 3u, 1u) + + std::max(depth >> 4u, 1u) && + TextureBuildGpuTileInfos(total.size, regions, layout, levels, + infos) && + infos.size() == 6; + const u32 table = std::countr_zero(tail.bytes); + for (u32 level = 0; level < levels && valid; level++) { + const auto &info = infos[level == 0 ? 0 : level + 1]; + valid &= info.tail && info.tail_x == volume_tail_xy[table][level][0] && + info.tail_y == volume_tail_xy[table][level][1] && + info.tiled_offset == 0; + } + if (valid) { + valid = infos[1].tail && + infos[1].tail_x == volume_tail_xy[table][0][0] && + infos[1].tail_y == volume_tail_xy[table][0][1] && + infos[1].tiled_offset == 4096; + } + Require( + name, "3D mip tail infos", valid, + "Standard4KB3D mip tail coordinates or block-slice packing changed"); + check_round_trip("3D mip tail", total.size, infos); + } + + for (const auto family : + {TileBlockFamily::Standard4KB, TileBlockFamily::Standard64KB, + TileBlockFamily::Prt64KB, TileBlockFamily::RenderTarget64KB, + TileBlockFamily::Depth64KB}) { + for (u32 bpe = 1; bpe <= 16; bpe <<= 1u) { + if (family == TileBlockFamily::Depth64KB && bpe == 16) + continue; + TileBlockLayout block{}; + Require(name, "tail layout", TileGetBlockLayout(family, bpe, block), + "tail family/BPE has no block layout"); + const u32 width = std::max(block.block_width / 4u, 1u); + const u32 height = std::max(block.block_height / 4u, 1u); + const u32 x = block.block_width / 2u; + const u32 y = block.block_height / 2u; + const u32 pitch = width; + const uint64_t linear_size = + static_cast(pitch) * height * bpe; + std::vector tiled(block.block_size); + std::vector cpu(linear_size, 0xcd); + std::vector gpu(linear_size, 0xab); + fill(&tiled, ++case_index); + GpuTileInfo info{}; + info.family = block.family; + info.bytes_per_element = block.bytes_per_element; + info.linear_size = linear_size; + info.tiled_size = block.block_size; + info.width = width; + info.height = height; + info.pitch = pitch; + info.tail = true; + info.tail_x = x; + info.tail_y = y; + info.surface_z = family == TileBlockFamily::RenderTarget64KB || + family == TileBlockFamily::Depth64KB + ? 2 + : 0; + convert_reference(false, &cpu, tiled, info); + gpu_detile(tiled, &gpu, block.block_size, linear_size, + std::span(&info, 1)); + compare("tail bytes", cpu, gpu); + + std::vector linear(linear_size); + std::vector cpu_tiled(block.block_size, 0xab); + std::vector gpu_tiled(block.block_size, 0xab); + fill(&linear, 0x400u + case_index); + convert_reference(true, &cpu_tiled, linear, info); + gpu_tile(linear, &gpu_tiled, block.block_size, linear_size, + std::span(&info, 1)); + compare("tail tile bytes", cpu_tiled, gpu_tiled); + } + } + + TileBlockLayout small_block{}; + Require(name, "small layout", + TileGetBlockLayout(TileBlockFamily::Standard256B, 4, small_block), + "small fixture layout is unavailable"); + std::vector small_input(small_block.block_size); + std::vector small_expected(small_block.block_size, 0); + std::vector small_output(small_block.block_size, 0xab); + fill(&small_input, 0xee); + GpuTileInfo small_info{}; + small_info.family = small_block.family; + small_info.bytes_per_element = small_block.bytes_per_element; + small_info.linear_size = small_output.size(); + small_info.tiled_size = small_input.size(); + small_info.width = 1; + small_info.height = 1; + small_info.pitch = small_block.block_width; + convert_reference(false, &small_expected, small_input, small_info); + gpu_detile(small_input, &small_output, small_input.size(), + small_output.size(), + std::span(&small_info, 1)); + compare("small detile", small_expected, small_output); + + std::fill(small_output.begin(), small_output.end(), 0xab); + gpu_detile(small_input, &small_output, small_input.size(), + small_output.size(), + std::span(&small_info, 1)); + compare("scheduler-owned reuse", small_expected, small_output); + + constexpr auto volume_format = Prospero::BufferFormat::k32UInt; + constexpr auto volume_tile = Prospero::TileMode::kLinear; + constexpr u32 volume_width = 8, volume_height = 4, volume_depth = 5; + constexpr u32 volume_levels = 3; + const u32 volume_pitch = + TileGetTexturePitch(volume_format, volume_width, volume_tile); + TileSizeAlign volume_size{}; + TileGetTextureTotalSize(volume_format, volume_width, volume_height, + volume_depth, volume_levels, volume_tile, true, + volume_size); + const auto volume_layout = TextureCalcUploadLayout( + volume_format, volume_width, volume_height, volume_levels, volume_depth, + volume_tile, volume_size.size, true, true, name); + const auto volume_copies = TextureBuildImageCopies(volume_layout); + std::vector volume_source(volume_size.size / sizeof(u32), 0); + std::vector> volume_probes; + for (const auto © : volume_copies) { + for (u32 y = 0; y < copy.imageExtent.height; y++) { + for (u32 x = 0; x < copy.imageExtent.width; x++) { + const auto index = + (copy.bufferOffset + + (static_cast(y) * copy.bufferRowLength + x) * + sizeof(u32)) / + sizeof(u32); + const u32 value = + 0xa0000000u | (copy.imageSubresource.mipLevel << 24u) | + (static_cast(copy.imageOffset.z) << 16u) | (y << 8u) | x; + Require(name, "linear volume native bounds", + index < volume_source.size(), + "linear volume copy escaped its padded guest layout"); + volume_source[index] = value; + volume_probes.emplace_back(static_cast(index), value); + } + } + } + auto volume_upload = + CreateHostBuffer(name, volume_size.size, + vk::BufferUsageFlagBits::eTransferSrc, volume_source); + auto volume_download = CreateHostBuffer( + name, volume_size.size, vk::BufferUsageFlagBits::eTransferDst, + std::vector(volume_source.size(), 0)); + ImageInfo volume_info{}; + volume_info.data = {0x10000, volume_size.size}; + volume_info.pixel_format = vk::Format::eR32Uint; + volume_info.guest_format = volume_format; + volume_info.type = Prospero::ImageType::kColor3D; + volume_info.extent = {volume_width, volume_height, volume_depth}; + volume_info.resources = {volume_levels, 1}; + volume_info.pitch = volume_pitch; + volume_info.bytes_per_block = sizeof(u32); + volume_info.tile_mode = volume_tile; + for (u32 level = 0; level < volume_levels; level++) { + volume_info.mip_layout[level] = {volume_layout.mips[level].offset, + volume_layout.mips[level].size, + volume_layout.mips[level].row_length, + volume_layout.mips[level].image_height}; + } + Libs::Graphics::Image volume_image(m_runtime_context, scheduler, + volume_info); + volume_image.Upload(volume_copies, volume_upload.buffer, 0, + volume_size.size); + volume_image.Download(volume_copies, volume_download.buffer, 0, + volume_size.size); + scheduler.Finish(); + const auto volume_observed = + ReadBuffer(name, volume_download, volume_source.size()); + Require(name, "linear volume native content", + std::ranges::all_of(volume_probes, + [&](const auto &probe) { + return volume_observed[probe.first] == + probe.second; + }), + "native upload/download did not preserve padded 3D mip rows"); + DestroyBuffer(&volume_download); + DestroyBuffer(&volume_upload); + std::printf("[gpu] %-32s ok (%u cases, %u format/mode pairs)\n", name, + case_index, format_cases); + } private: - RenderContext& Renderer() { - EXIT_IF(m_renderer == nullptr); - return *m_renderer; - } + RenderContext &Renderer() { + EXIT_IF(m_renderer == nullptr); + return *m_renderer; + } - void EnsureRuntimeContext() { - if (m_runtime_context.allocator != nullptr) { - return; - } - m_runtime_context.instance = m_instance; - m_runtime_context.physical_device = m_physical_device; - m_runtime_context.device = m_device; - m_physical_device.getProperties(&m_runtime_context.physical_device_properties); - m_runtime_context.physical_device_memory_properties = m_memory_properties; - m_runtime_context.queue_family = m_queue_family; - m_runtime_context.queue = m_queue; + void EnsureRuntimeContext() { + if (m_runtime_context.allocator != nullptr) { + return; + } + m_runtime_context.instance = m_instance; + m_runtime_context.physical_device = m_physical_device; + m_runtime_context.device = m_device; + m_physical_device.getProperties( + &m_runtime_context.physical_device_properties); + m_runtime_context.physical_device_memory_properties = m_memory_properties; + m_runtime_context.queue_family = m_queue_family; + m_runtime_context.queue = m_queue; - VmaVulkanFunctions functions {}; - functions.vkGetInstanceProcAddr = VULKAN_HPP_DEFAULT_DISPATCHER.vkGetInstanceProcAddr; - functions.vkGetDeviceProcAddr = VULKAN_HPP_DEFAULT_DISPATCHER.vkGetDeviceProcAddr; + VmaVulkanFunctions functions{}; + functions.vkGetInstanceProcAddr = + VULKAN_HPP_DEFAULT_DISPATCHER.vkGetInstanceProcAddr; + functions.vkGetDeviceProcAddr = + VULKAN_HPP_DEFAULT_DISPATCHER.vkGetDeviceProcAddr; - VmaAllocatorCreateInfo allocator_info {}; - allocator_info.instance = m_instance; - allocator_info.physicalDevice = m_physical_device; - allocator_info.device = m_device; - allocator_info.pVulkanFunctions = &functions; - allocator_info.vulkanApiVersion = VK_API_VERSION_1_3; - RequireVk("VulkanHarness", "runtime context", - static_cast( - vmaCreateAllocator(&allocator_info, &m_runtime_context.allocator)), - "vmaCreateAllocator"); - m_renderer = std::make_unique(m_runtime_context); - } + VmaAllocatorCreateInfo allocator_info{}; + allocator_info.instance = m_instance; + allocator_info.physicalDevice = m_physical_device; + allocator_info.device = m_device; + allocator_info.pVulkanFunctions = &functions; + allocator_info.vulkanApiVersion = VK_API_VERSION_1_3; + RequireVk("VulkanHarness", "runtime context", + static_cast(vmaCreateAllocator( + &allocator_info, &m_runtime_context.allocator)), + "vmaCreateAllocator"); + m_renderer = std::make_unique(m_runtime_context); + } - void Init() { - static vk::detail::DynamicLoader loader; - const auto get_instance_proc_addr = - loader.getProcAddress("vkGetInstanceProcAddr"); - Require("VulkanHarness", "dispatch", get_instance_proc_addr != nullptr, - "could not load the Vulkan loader"); - VULKAN_HPP_DEFAULT_DISPATCHER.init(get_instance_proc_addr); + void Init() { + static vk::detail::DynamicLoader loader; + const auto get_instance_proc_addr = + loader.getProcAddress( + "vkGetInstanceProcAddr"); + Require("VulkanHarness", "dispatch", get_instance_proc_addr != nullptr, + "could not load the Vulkan loader"); + VULKAN_HPP_DEFAULT_DISPATCHER.init(get_instance_proc_addr); - vk::ApplicationInfo app {}; - app.sType = vk::StructureType::eApplicationInfo; - app.pApplicationName = "ShaderRecompilerComputeTests"; - app.apiVersion = VK_API_VERSION_1_3; + vk::ApplicationInfo app{}; + app.sType = vk::StructureType::eApplicationInfo; + app.pApplicationName = "ShaderRecompilerComputeTests"; + app.apiVersion = VK_API_VERSION_1_3; - vk::InstanceCreateInfo instance_info {}; - instance_info.sType = vk::StructureType::eInstanceCreateInfo; - instance_info.pApplicationInfo = &app; - RequireVk("VulkanHarness", "dispatch", - vk::createInstance(&instance_info, nullptr, &m_instance), "vkCreateInstance"); - VULKAN_HPP_DEFAULT_DISPATCHER.init(m_instance); + vk::InstanceCreateInfo instance_info{}; + instance_info.sType = vk::StructureType::eInstanceCreateInfo; + instance_info.pApplicationInfo = &app; + RequireVk("VulkanHarness", "dispatch", + vk::createInstance(&instance_info, nullptr, &m_instance), + "vkCreateInstance"); + VULKAN_HPP_DEFAULT_DISPATCHER.init(m_instance); - u32 physical_count = 0; - RequireVk("VulkanHarness", "dispatch", - m_instance.enumeratePhysicalDevices(&physical_count, nullptr), - "vkEnumeratePhysicalDevices"); - Require("VulkanHarness", "dispatch", physical_count != 0, "no Vulkan physical devices"); - std::vector physical_devices(physical_count); - RequireVk("VulkanHarness", "dispatch", - m_instance.enumeratePhysicalDevices(&physical_count, physical_devices.data()), - "vkEnumeratePhysicalDevices"); + u32 physical_count = 0; + RequireVk("VulkanHarness", "dispatch", + m_instance.enumeratePhysicalDevices(&physical_count, nullptr), + "vkEnumeratePhysicalDevices"); + Require("VulkanHarness", "dispatch", physical_count != 0, + "no Vulkan physical devices"); + std::vector physical_devices(physical_count); + RequireVk("VulkanHarness", "dispatch", + m_instance.enumeratePhysicalDevices(&physical_count, + physical_devices.data()), + "vkEnumeratePhysicalDevices"); - for (auto physical: physical_devices) { - u32 queue_count = 0; - physical.getQueueFamilyProperties(&queue_count, nullptr); - std::vector queues(queue_count); - physical.getQueueFamilyProperties(&queue_count, queues.data()); - for (u32 i = 0; i < queue_count; i++) { - if ((queues[i].queueFlags & - (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) == - (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) { - m_physical_device = physical; - m_queue_family = i; - break; - } - } - if (m_physical_device != nullptr) { - break; - } - } - Require("VulkanHarness", "dispatch", m_physical_device != nullptr, - "no Vulkan graphics+compute queue family"); - m_physical_device.getMemoryProperties(&m_memory_properties); + for (auto physical : physical_devices) { + u32 queue_count = 0; + physical.getQueueFamilyProperties(&queue_count, nullptr); + std::vector queues(queue_count); + physical.getQueueFamilyProperties(&queue_count, queues.data()); + for (u32 i = 0; i < queue_count; i++) { + if ((queues[i].queueFlags & + (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) == + (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) { + m_physical_device = physical; + m_queue_family = i; + break; + } + } + if (m_physical_device != nullptr) { + break; + } + } + Require("VulkanHarness", "dispatch", m_physical_device != nullptr, + "no Vulkan graphics+compute queue family"); + m_physical_device.getMemoryProperties(&m_memory_properties); - vk::PhysicalDeviceFeatures available_features {}; - m_physical_device.getFeatures(&available_features); - vk::PhysicalDeviceVulkan12Features available_features12 {}; - available_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; - vk::PhysicalDeviceVulkan13Features available_features13 {}; - available_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features; - available_features13.pNext = &available_features12; - vk::PhysicalDeviceFeatures2 available_features2 {}; - available_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2; - available_features2.pNext = &available_features13; - m_physical_device.getFeatures2(&available_features2); - Require("VulkanHarness", "dispatch", - available_features.shaderStorageImageWriteWithoutFormat == true, - "shaderStorageImageWriteWithoutFormat is not supported"); - Require("VulkanHarness", "dispatch", - available_features.shaderStorageImageReadWithoutFormat == true, - "shaderStorageImageReadWithoutFormat is not supported"); - Require("VulkanHarness", "dispatch", available_features12.timelineSemaphore == true, - "timeline semaphores are not supported"); - Require("VulkanHarness", "dispatch", available_features13.dynamicRendering == true, - "dynamic rendering is not supported"); - Require("VulkanHarness", "dispatch", available_features13.synchronization2 == true, - "synchronization2 is not supported"); - Require("VulkanHarness", "dispatch", available_features.sampleRateShading == true, - "sample-rate shading is not supported"); + vk::PhysicalDeviceFeatures available_features{}; + m_physical_device.getFeatures(&available_features); + vk::PhysicalDeviceVulkan12Features available_features12{}; + available_features12.sType = + vk::StructureType::ePhysicalDeviceVulkan12Features; + vk::PhysicalDeviceVulkan13Features available_features13{}; + available_features13.sType = + vk::StructureType::ePhysicalDeviceVulkan13Features; + available_features13.pNext = &available_features12; + vk::PhysicalDeviceFeatures2 available_features2{}; + available_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2; + available_features2.pNext = &available_features13; + m_physical_device.getFeatures2(&available_features2); + Require("VulkanHarness", "dispatch", + available_features.shaderStorageImageWriteWithoutFormat == true, + "shaderStorageImageWriteWithoutFormat is not supported"); + Require("VulkanHarness", "dispatch", + available_features.shaderStorageImageReadWithoutFormat == true, + "shaderStorageImageReadWithoutFormat is not supported"); + Require("VulkanHarness", "dispatch", + available_features12.timelineSemaphore == true, + "timeline semaphores are not supported"); + Require("VulkanHarness", "dispatch", + available_features13.dynamicRendering == true, + "dynamic rendering is not supported"); + Require("VulkanHarness", "dispatch", + available_features13.synchronization2 == true, + "synchronization2 is not supported"); + Require("VulkanHarness", "dispatch", + available_features.sampleRateShading == true, + "sample-rate shading is not supported"); - float priority = 1.0f; - vk::DeviceQueueCreateInfo queue_info {}; - queue_info.sType = vk::StructureType::eDeviceQueueCreateInfo; - queue_info.queueFamilyIndex = m_queue_family; - queue_info.queueCount = 1; - queue_info.pQueuePriorities = &priority; + float priority = 1.0f; + vk::DeviceQueueCreateInfo queue_info{}; + queue_info.sType = vk::StructureType::eDeviceQueueCreateInfo; + queue_info.queueFamilyIndex = m_queue_family; + queue_info.queueCount = 1; + queue_info.pQueuePriorities = &priority; - vk::DeviceCreateInfo device_info {}; - device_info.sType = vk::StructureType::eDeviceCreateInfo; - device_info.queueCreateInfoCount = 1; - device_info.pQueueCreateInfos = &queue_info; - vk::PhysicalDeviceVulkan12Features device_features12 {}; - device_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; - device_features12.timelineSemaphore = true; - vk::PhysicalDeviceVulkan13Features device_features13 {}; - device_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features; - device_features13.pNext = &device_features12; - device_features13.dynamicRendering = true; - device_features13.synchronization2 = true; - device_info.pNext = &device_features13; - vk::PhysicalDeviceFeatures device_features {}; - device_features.shaderStorageImageWriteWithoutFormat = true; - device_features.shaderStorageImageReadWithoutFormat = true; - device_features.sampleRateShading = true; - device_info.pEnabledFeatures = &device_features; - constexpr const char* device_extensions[] = {VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME}; - device_info.enabledExtensionCount = std::size(device_extensions); - device_info.ppEnabledExtensionNames = device_extensions; - RequireVk("VulkanHarness", "dispatch", - m_physical_device.createDevice(&device_info, nullptr, &m_device), - "vkCreateDevice"); - VULKAN_HPP_DEFAULT_DISPATCHER.init(m_device); - m_device.getQueue(m_queue_family, 0, &m_queue); + vk::DeviceCreateInfo device_info{}; + device_info.sType = vk::StructureType::eDeviceCreateInfo; + device_info.queueCreateInfoCount = 1; + device_info.pQueueCreateInfos = &queue_info; + vk::PhysicalDeviceVulkan12Features device_features12{}; + device_features12.sType = + vk::StructureType::ePhysicalDeviceVulkan12Features; + device_features12.timelineSemaphore = true; + vk::PhysicalDeviceVulkan13Features device_features13{}; + device_features13.sType = + vk::StructureType::ePhysicalDeviceVulkan13Features; + device_features13.pNext = &device_features12; + device_features13.dynamicRendering = true; + device_features13.synchronization2 = true; + device_info.pNext = &device_features13; + vk::PhysicalDeviceFeatures device_features{}; + device_features.shaderStorageImageWriteWithoutFormat = true; + device_features.shaderStorageImageReadWithoutFormat = true; + device_features.sampleRateShading = true; + device_info.pEnabledFeatures = &device_features; + constexpr const char *device_extensions[] = { + VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME}; + device_info.enabledExtensionCount = std::size(device_extensions); + device_info.ppEnabledExtensionNames = device_extensions; + RequireVk("VulkanHarness", "dispatch", + m_physical_device.createDevice(&device_info, nullptr, &m_device), + "vkCreateDevice"); + VULKAN_HPP_DEFAULT_DISPATCHER.init(m_device); + m_device.getQueue(m_queue_family, 0, &m_queue); - vk::CommandPoolCreateInfo pool_info {}; - pool_info.sType = vk::StructureType::eCommandPoolCreateInfo; - pool_info.queueFamilyIndex = m_queue_family; - pool_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer; - RequireVk("VulkanHarness", "dispatch", - m_device.createCommandPool(&pool_info, nullptr, &m_command_pool), - "vkCreateCommandPool"); - } + vk::CommandPoolCreateInfo pool_info{}; + pool_info.sType = vk::StructureType::eCommandPoolCreateInfo; + pool_info.queueFamilyIndex = m_queue_family; + pool_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer; + RequireVk("VulkanHarness", "dispatch", + m_device.createCommandPool(&pool_info, nullptr, &m_command_pool), + "vkCreateCommandPool"); + } - void Destroy() { - if (m_device != nullptr) { - RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle"); - if (m_runtime_context.allocator != nullptr) { - m_renderer.reset(); - vmaDestroyAllocator(m_runtime_context.allocator); - m_runtime_context.allocator = nullptr; - } - if (m_command_pool != nullptr) { - m_device.destroyCommandPool(m_command_pool, nullptr); - } - m_device.destroy(nullptr); - } - if (m_instance != nullptr) { - m_instance.destroy(nullptr); - } - } + void Destroy() { + if (m_device != nullptr) { + RequireVulkanSuccess(m_device.waitIdle(), "vkDeviceWaitIdle"); + if (m_runtime_context.allocator != nullptr) { + m_renderer.reset(); + vmaDestroyAllocator(m_runtime_context.allocator); + m_runtime_context.allocator = nullptr; + } + if (m_command_pool != nullptr) { + m_device.destroyCommandPool(m_command_pool, nullptr); + } + m_device.destroy(nullptr); + } + if (m_instance != nullptr) { + m_instance.destroy(nullptr); + } + } - bool FindMemoryType(u32 type_bits, vk::MemoryPropertyFlags required, u32* index) const { - for (u32 i = 0; i < m_memory_properties.memoryTypeCount; i++) { - if ((type_bits & (1u << i)) == 0) { - continue; - } - if ((m_memory_properties.memoryTypes[i].propertyFlags & required) == required) { - *index = i; - return true; - } - } - return false; - } + bool FindMemoryType(u32 type_bits, vk::MemoryPropertyFlags required, + u32 *index) const { + for (u32 i = 0; i < m_memory_properties.memoryTypeCount; i++) { + if ((type_bits & (1u << i)) == 0) { + continue; + } + if ((m_memory_properties.memoryTypes[i].propertyFlags & required) == + required) { + *index = i; + return true; + } + } + return false; + } - static vk::AccessFlags AccessForLayout(vk::ImageLayout layout) { - switch (layout) { - case vk::ImageLayout::eTransferDstOptimal: return vk::AccessFlagBits::eTransferWrite; - case vk::ImageLayout::eTransferSrcOptimal: return vk::AccessFlagBits::eTransferRead; - case vk::ImageLayout::eShaderReadOnlyOptimal: return vk::AccessFlagBits::eShaderRead; - case vk::ImageLayout::eGeneral: - return vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; - default: return {}; - } - } + static vk::AccessFlags AccessForLayout(vk::ImageLayout layout) { + switch (layout) { + case vk::ImageLayout::eTransferDstOptimal: + return vk::AccessFlagBits::eTransferWrite; + case vk::ImageLayout::eTransferSrcOptimal: + return vk::AccessFlagBits::eTransferRead; + case vk::ImageLayout::eShaderReadOnlyOptimal: + return vk::AccessFlagBits::eShaderRead; + case vk::ImageLayout::eGeneral: + return vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite; + default: + return {}; + } + } - vk::CommandBuffer BeginCommands(const char* shader_name, const char* stage) { - vk::CommandBufferAllocateInfo cmd_alloc {}; - cmd_alloc.sType = vk::StructureType::eCommandBufferAllocateInfo; - cmd_alloc.commandPool = m_command_pool; - cmd_alloc.level = vk::CommandBufferLevel::ePrimary; - cmd_alloc.commandBufferCount = 1; - vk::CommandBuffer cmd = nullptr; - RequireVk(shader_name, stage, m_device.allocateCommandBuffers(&cmd_alloc, &cmd), - "vkAllocateCommandBuffers"); + vk::CommandBuffer BeginCommands(const char *shader_name, const char *stage) { + vk::CommandBufferAllocateInfo cmd_alloc{}; + cmd_alloc.sType = vk::StructureType::eCommandBufferAllocateInfo; + cmd_alloc.commandPool = m_command_pool; + cmd_alloc.level = vk::CommandBufferLevel::ePrimary; + cmd_alloc.commandBufferCount = 1; + vk::CommandBuffer cmd = nullptr; + RequireVk(shader_name, stage, + m_device.allocateCommandBuffers(&cmd_alloc, &cmd), + "vkAllocateCommandBuffers"); - vk::CommandBufferBeginInfo begin {}; - begin.sType = vk::StructureType::eCommandBufferBeginInfo; - begin.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit; - RequireVk(shader_name, stage, cmd.begin(&begin), "vkBeginCommandBuffer"); - return cmd; - } + vk::CommandBufferBeginInfo begin{}; + begin.sType = vk::StructureType::eCommandBufferBeginInfo; + begin.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit; + RequireVk(shader_name, stage, cmd.begin(&begin), "vkBeginCommandBuffer"); + return cmd; + } - void EndSubmitAndFree(const char* shader_name, const char* stage, vk::CommandBuffer cmd) { - RequireVk(shader_name, stage, cmd.end(), "vkEndCommandBuffer"); + void EndSubmitAndFree(const char *shader_name, const char *stage, + vk::CommandBuffer cmd) { + RequireVk(shader_name, stage, cmd.end(), "vkEndCommandBuffer"); - vk::FenceCreateInfo fence_info {}; - fence_info.sType = vk::StructureType::eFenceCreateInfo; - vk::Fence fence = nullptr; - RequireVk(shader_name, stage, m_device.createFence(&fence_info, nullptr, &fence), - "vkCreateFence"); + vk::FenceCreateInfo fence_info{}; + fence_info.sType = vk::StructureType::eFenceCreateInfo; + vk::Fence fence = nullptr; + RequireVk(shader_name, stage, + m_device.createFence(&fence_info, nullptr, &fence), + "vkCreateFence"); - vk::SubmitInfo submit {}; - submit.sType = vk::StructureType::eSubmitInfo; - submit.commandBufferCount = 1; - submit.pCommandBuffers = &cmd; - RequireVk(shader_name, stage, m_queue.submit(1, &submit, fence), "vkQueueSubmit"); - RequireVk(shader_name, stage, m_device.waitForFences(1, &fence, true, UINT64_MAX), - "vkWaitForFences"); + vk::SubmitInfo submit{}; + submit.sType = vk::StructureType::eSubmitInfo; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &cmd; + RequireVk(shader_name, stage, m_queue.submit(1, &submit, fence), + "vkQueueSubmit"); + RequireVk(shader_name, stage, + m_device.waitForFences(1, &fence, true, UINT64_MAX), + "vkWaitForFences"); - m_device.destroyFence(fence, nullptr); - m_device.freeCommandBuffers(m_command_pool, 1, &cmd); - } + m_device.destroyFence(fence, nullptr); + m_device.freeCommandBuffers(m_command_pool, 1, &cmd); + } - void AddImageBarrier(vk::CommandBuffer cmd, vk::Image image, vk::ImageLayout old_layout, - vk::ImageLayout new_layout, vk::PipelineStageFlags src_stage, - vk::PipelineStageFlags dst_stage, vk::AccessFlags src_access, - vk::AccessFlags dst_access, u32 mip_levels = 1, u32 layers = 1) { - vk::ImageMemoryBarrier barrier {}; - barrier.sType = vk::StructureType::eImageMemoryBarrier; - barrier.srcAccessMask = src_access; - barrier.dstAccessMask = dst_access; - barrier.oldLayout = old_layout; - barrier.newLayout = new_layout; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = image; - barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - barrier.subresourceRange.baseMipLevel = 0; - barrier.subresourceRange.levelCount = mip_levels; - barrier.subresourceRange.baseArrayLayer = 0; - barrier.subresourceRange.layerCount = layers; - cmd.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 0, nullptr, 1, &barrier); - } + void AddImageBarrier(vk::CommandBuffer cmd, vk::Image image, + vk::ImageLayout old_layout, vk::ImageLayout new_layout, + vk::PipelineStageFlags src_stage, + vk::PipelineStageFlags dst_stage, + vk::AccessFlags src_access, vk::AccessFlags dst_access, + u32 mip_levels = 1, u32 layers = 1) { + vk::ImageMemoryBarrier barrier{}; + barrier.sType = vk::StructureType::eImageMemoryBarrier; + barrier.srcAccessMask = src_access; + barrier.dstAccessMask = dst_access; + barrier.oldLayout = old_layout; + barrier.newLayout = new_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; + barrier.subresourceRange.baseMipLevel = 0; + barrier.subresourceRange.levelCount = mip_levels; + barrier.subresourceRange.baseArrayLayer = 0; + barrier.subresourceRange.layerCount = layers; + cmd.pipelineBarrier(src_stage, dst_stage, {}, 0, nullptr, 0, nullptr, 1, + &barrier); + } - void TransitionImage(const char* shader_name, Image* image, vk::ImageLayout new_layout, - vk::PipelineStageFlags src_stage, vk::PipelineStageFlags dst_stage, - vk::AccessFlags src_access, vk::AccessFlags dst_access) { - vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch"); - AddImageBarrier(cmd, image->image, image->layout, new_layout, src_stage, dst_stage, - src_access, dst_access, image->mip_levels, image->layers); - EndSubmitAndFree(shader_name, "dispatch", cmd); - image->layout = new_layout; - } + void TransitionImage(const char *shader_name, Image *image, + vk::ImageLayout new_layout, + vk::PipelineStageFlags src_stage, + vk::PipelineStageFlags dst_stage, + vk::AccessFlags src_access, vk::AccessFlags dst_access) { + vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch"); + AddImageBarrier(cmd, image->image, image->layout, new_layout, src_stage, + dst_stage, src_access, dst_access, image->mip_levels, + image->layers); + EndSubmitAndFree(shader_name, "dispatch", cmd); + image->layout = new_layout; + } - void UploadImage(const char* shader_name, Image* image, vk::Buffer staging, - vk::ImageLayout final_layout) { - UploadImageMips(shader_name, image, staging, final_layout); - } + void UploadImage(const char *shader_name, Image *image, vk::Buffer staging, + vk::ImageLayout final_layout) { + UploadImageMips(shader_name, image, staging, final_layout); + } - void UploadImageMips(const char* shader_name, Image* image, vk::Buffer staging, - vk::ImageLayout final_layout) { - vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch"); - AddImageBarrier(cmd, image->image, vk::ImageLayout::eUndefined, - vk::ImageLayout::eTransferDstOptimal, vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eTransfer, {}, - vk::AccessFlagBits::eTransferWrite, image->mip_levels, image->layers); - std::vector copies; - copies.reserve(image->mip_levels); - vk::DeviceSize offset = 0; - for (u32 level = 0; level < image->mip_levels; level++) { - vk::BufferImageCopy copy {}; - copy.bufferOffset = offset; - copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - copy.imageSubresource.mipLevel = level; - copy.imageSubresource.baseArrayLayer = 0; - copy.imageSubresource.layerCount = image->layers; - copy.imageExtent.width = MipExtent(image->width, level); - copy.imageExtent.height = MipExtent(image->height, level); - copy.imageExtent.depth = 1; - copies.push_back(copy); - offset += static_cast(ImageMipDwordCount(image->width, image->height, - image->dwords_per_pixel, level, - image->layers) * - sizeof(u32)); - } - cmd.copyBufferToImage(staging, image->image, vk::ImageLayout::eTransferDstOptimal, - static_cast(copies.size()), copies.data()); - AddImageBarrier(cmd, image->image, vk::ImageLayout::eTransferDstOptimal, final_layout, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eComputeShader, - vk::AccessFlagBits::eTransferWrite, AccessForLayout(final_layout), - image->mip_levels, image->layers); - EndSubmitAndFree(shader_name, "dispatch", cmd); - image->layout = final_layout; - } + void UploadImageMips(const char *shader_name, Image *image, + vk::Buffer staging, vk::ImageLayout final_layout) { + vk::CommandBuffer cmd = BeginCommands(shader_name, "dispatch"); + AddImageBarrier(cmd, image->image, vk::ImageLayout::eUndefined, + vk::ImageLayout::eTransferDstOptimal, + vk::PipelineStageFlagBits::eTopOfPipe, + vk::PipelineStageFlagBits::eTransfer, {}, + vk::AccessFlagBits::eTransferWrite, image->mip_levels, + image->layers); + std::vector copies; + copies.reserve(image->mip_levels); + vk::DeviceSize offset = 0; + for (u32 level = 0; level < image->mip_levels; level++) { + vk::BufferImageCopy copy{}; + copy.bufferOffset = offset; + copy.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + copy.imageSubresource.mipLevel = level; + copy.imageSubresource.baseArrayLayer = 0; + copy.imageSubresource.layerCount = image->layers; + copy.imageExtent.width = MipExtent(image->width, level); + copy.imageExtent.height = MipExtent(image->height, level); + copy.imageExtent.depth = 1; + copies.push_back(copy); + offset += static_cast( + ImageMipDwordCount(image->width, image->height, + image->dwords_per_pixel, level, image->layers) * + sizeof(u32)); + } + cmd.copyBufferToImage(staging, image->image, + vk::ImageLayout::eTransferDstOptimal, + static_cast(copies.size()), copies.data()); + AddImageBarrier(cmd, image->image, vk::ImageLayout::eTransferDstOptimal, + final_layout, vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eComputeShader, + vk::AccessFlagBits::eTransferWrite, + AccessForLayout(final_layout), image->mip_levels, + image->layers); + EndSubmitAndFree(shader_name, "dispatch", cmd); + image->layout = final_layout; + } - Buffer CreateHostBuffer(const char* shader_name, vk::DeviceSize size, - vk::BufferUsageFlags usage, const std::vector& contents) { - Buffer ret; - ret.size = std::max(size, sizeof(u32)); + Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size, + vk::BufferUsageFlags usage, + const std::vector &contents) { + Buffer ret; + ret.size = std::max(size, sizeof(u32)); - vk::BufferCreateInfo buffer_info {}; - buffer_info.sType = vk::StructureType::eBufferCreateInfo; - buffer_info.size = ret.size; - buffer_info.usage = usage; - buffer_info.sharingMode = vk::SharingMode::eExclusive; - RequireVk(shader_name, "dispatch", - m_device.createBuffer(&buffer_info, nullptr, &ret.buffer), "vkCreateBuffer"); + vk::BufferCreateInfo buffer_info{}; + buffer_info.sType = vk::StructureType::eBufferCreateInfo; + buffer_info.size = ret.size; + buffer_info.usage = usage; + buffer_info.sharingMode = vk::SharingMode::eExclusive; + RequireVk(shader_name, "dispatch", + m_device.createBuffer(&buffer_info, nullptr, &ret.buffer), + "vkCreateBuffer"); - vk::MemoryRequirements req {}; - m_device.getBufferMemoryRequirements(ret.buffer, &req); - u32 memory_type = 0; - ret.coherent = FindMemoryType(req.memoryTypeBits, - vk::MemoryPropertyFlagBits::eHostVisible | - vk::MemoryPropertyFlagBits::eHostCoherent, - &memory_type); - if (!ret.coherent) { - Require(shader_name, "dispatch", - FindMemoryType(req.memoryTypeBits, vk::MemoryPropertyFlagBits::eHostVisible, - &memory_type), - "no host-visible memory type for staging buffer"); - } + vk::MemoryRequirements req{}; + m_device.getBufferMemoryRequirements(ret.buffer, &req); + u32 memory_type = 0; + ret.coherent = FindMemoryType(req.memoryTypeBits, + vk::MemoryPropertyFlagBits::eHostVisible | + vk::MemoryPropertyFlagBits::eHostCoherent, + &memory_type); + if (!ret.coherent) { + Require(shader_name, "dispatch", + FindMemoryType(req.memoryTypeBits, + vk::MemoryPropertyFlagBits::eHostVisible, + &memory_type), + "no host-visible memory type for staging buffer"); + } - vk::MemoryAllocateInfo alloc {}; - alloc.sType = vk::StructureType::eMemoryAllocateInfo; - alloc.allocationSize = req.size; - alloc.memoryTypeIndex = memory_type; - RequireVk(shader_name, "dispatch", m_device.allocateMemory(&alloc, nullptr, &ret.memory), - "vkAllocateMemory"); - RequireVk(shader_name, "dispatch", m_device.bindBufferMemory(ret.buffer, ret.memory, 0), - "vkBindBufferMemory"); - if (!contents.empty()) { - WriteBuffer(shader_name, ret, contents); - } - return ret; - } + vk::MemoryAllocateInfo alloc{}; + alloc.sType = vk::StructureType::eMemoryAllocateInfo; + alloc.allocationSize = req.size; + alloc.memoryTypeIndex = memory_type; + RequireVk(shader_name, "dispatch", + m_device.allocateMemory(&alloc, nullptr, &ret.memory), + "vkAllocateMemory"); + RequireVk(shader_name, "dispatch", + m_device.bindBufferMemory(ret.buffer, ret.memory, 0), + "vkBindBufferMemory"); + if (!contents.empty()) { + WriteBuffer(shader_name, ret, contents); + } + return ret; + } - void WriteBuffer(const char* shader_name, const Buffer& buffer, - const std::vector& contents) { - void* data = nullptr; - RequireVk(shader_name, "dispatch", - m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data), "vkMapMemory"); - std::memcpy(data, contents.data(), contents.size() * sizeof(u32)); - if (!buffer.coherent) { - vk::MappedMemoryRange range {}; - range.sType = vk::StructureType::eMappedMemoryRange; - range.memory = buffer.memory; - range.offset = 0; - range.size = VK_WHOLE_SIZE; - RequireVk(shader_name, "dispatch", m_device.flushMappedMemoryRanges(1, &range), - "vkFlushMappedMemoryRanges"); - } - m_device.unmapMemory(buffer.memory); - } + void WriteBuffer(const char *shader_name, const Buffer &buffer, + const std::vector &contents) { + void *data = nullptr; + RequireVk(shader_name, "dispatch", + m_device.mapMemory(buffer.memory, 0, buffer.size, {}, &data), + "vkMapMemory"); + std::memcpy(data, contents.data(), contents.size() * sizeof(u32)); + if (!buffer.coherent) { + vk::MappedMemoryRange range{}; + range.sType = vk::StructureType::eMappedMemoryRange; + range.memory = buffer.memory; + range.offset = 0; + range.size = VK_WHOLE_SIZE; + RequireVk(shader_name, "dispatch", + m_device.flushMappedMemoryRanges(1, &range), + "vkFlushMappedMemoryRanges"); + } + m_device.unmapMemory(buffer.memory); + } - vk::Instance m_instance = nullptr; - vk::PhysicalDevice m_physical_device = nullptr; - vk::Device m_device = nullptr; - vk::Queue m_queue = nullptr; - vk::CommandPool m_command_pool = nullptr; - u32 m_queue_family = 0; - vk::PhysicalDeviceMemoryProperties m_memory_properties {}; - GraphicContext m_runtime_context {}; - std::unique_ptr m_renderer; + vk::Instance m_instance = nullptr; + vk::PhysicalDevice m_physical_device = nullptr; + vk::Device m_device = nullptr; + vk::Queue m_queue = nullptr; + vk::CommandPool m_command_pool = nullptr; + u32 m_queue_family = 0; + vk::PhysicalDeviceMemoryProperties m_memory_properties{}; + GraphicContext m_runtime_context{}; + std::unique_ptr m_renderer; }; -void CompareWords(const TestCase& test, const char* stage, const std::vector& expected, - const std::vector& actual) { - if (actual == expected) { - return; - } - std::ostringstream out; - out << "expected ["; - for (size_t i = 0; i < expected.size(); i++) { - out << (i == 0 ? "" : ", ") << Hex(expected[i]); - } - out << "] actual ["; - for (size_t i = 0; i < actual.size(); i++) { - out << (i == 0 ? "" : ", ") << Hex(actual[i]); - } - out << "]"; - Fail(test.name, stage, out.str()); +void CompareWords(const TestCase &test, const char *stage, + const std::vector &expected, + const std::vector &actual) { + if (actual == expected) { + return; + } + std::ostringstream out; + out << "expected ["; + for (size_t i = 0; i < expected.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(expected[i]); + } + out << "] actual ["; + for (size_t i = 0; i < actual.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(actual[i]); + } + out << "]"; + Fail(test.name, stage, out.str()); } -void CompareGraphicsWords(const GraphicsCase& test, const std::vector& actual) { - if (actual == test.expected_pixel) { - return; - } - std::ostringstream out; - out << "expected ["; - for (size_t i = 0; i < test.expected_pixel.size(); i++) { - out << (i == 0 ? "" : ", ") << Hex(test.expected_pixel[i]); - } - out << "] actual ["; - for (size_t i = 0; i < actual.size(); i++) { - out << (i == 0 ? "" : ", ") << Hex(actual[i]); - } - out << "]"; - Fail(test.name, "graphics readback", out.str()); +void CompareGraphicsWords(const GraphicsCase &test, + const std::vector &actual) { + if (actual == test.expected_pixel) { + return; + } + std::ostringstream out; + out << "expected ["; + for (size_t i = 0; i < test.expected_pixel.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(test.expected_pixel[i]); + } + out << "] actual ["; + for (size_t i = 0; i < actual.size(); i++) { + out << (i == 0 ? "" : ", ") << Hex(actual[i]); + } + out << "]"; + Fail(test.name, "graphics readback", out.str()); } -void RunCase(VulkanHarness* vulkan, const TestCase& test) { - auto compiled = CompileCase(test); - if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) { - Require(test.name, "resource specialization", - !compiled.program.info.images.empty() && - compiled.program.info.images[0].storage_swizzle == - test.image_descriptor_swizzle, - "storage image descriptor swizzle did not reach the specialized " - "program"); - } - if (test.compile_only) { - std::printf("[compute] %-32s ok\n", test.name); - return; - } - const auto dwords = - std::max({test.initial.size(), test.expected.size(), static_cast(1)}); - auto buffer = vulkan->CreateStorageBuffer(test.name, test.initial, dwords); +void RunCase(VulkanHarness *vulkan, const TestCase &test) { + auto compiled = CompileCase(test); + if (test.image_descriptor_swizzle != DstSel(4, 5, 6, 7)) { + Require(test.name, "resource specialization", + !compiled.program.info.images.empty() && + compiled.program.info.images[0].storage_swizzle == + test.image_descriptor_swizzle, + "storage image descriptor swizzle did not reach the specialized " + "program"); + } + if (test.compile_only) { + std::printf("[compute] %-32s ok\n", test.name); + return; + } + const auto dwords = std::max( + {test.initial.size(), test.expected.size(), static_cast(1)}); + auto buffer = vulkan->CreateStorageBuffer(test.name, test.initial, dwords); - using Kind = ShaderRecompiler::IR::DescriptorBindingKind; - auto Has = [&](Kind kind) { - return ShaderRecompiler::IR::FindBinding(compiled.program.bindings, kind) != nullptr; - }; - VulkanHarness::Image sampled_image; - VulkanHarness::Image storage_image; - VulkanHarness::Image storage_image_uint; - VulkanHarness::Buffer gds_buffer; - vk::Sampler sampler = nullptr; - const bool needs_sampled_image = Has(Kind::Sampled1D) || Has(Kind::Sampled1DArray) || - Has(Kind::Sampled2D) || Has(Kind::Sampled2DArray) || - Has(Kind::Sampled3D) || Has(Kind::SampledUint2D) || - Has(Kind::SampledUint1D) || Has(Kind::SampledUint1DArray) || - Has(Kind::SampledUint2DArray) || Has(Kind::SampledUint3D); - const bool needs_storage_image = Has(Kind::Storage1D) || Has(Kind::Storage1DArray) || - Has(Kind::Storage2D) || Has(Kind::Storage2DArray) || - Has(Kind::Storage3D) || Has(Kind::StorageUint2D) || - Has(Kind::StorageUint1D) || Has(Kind::StorageUint1DArray) || - Has(Kind::StorageUint2DArray) || Has(Kind::StorageUint3D); - const bool needs_sampler = Has(Kind::Samplers); - const bool needs_gds = Has(Kind::Gds); - if (needs_gds) { - const auto gds_dwords = - std::max({test.gds_initial.size(), test.expected_gds.size(), 1u}); - gds_buffer = vulkan->CreateStorageBuffer(test.name, test.gds_initial, gds_dwords); - } - if (needs_sampled_image) { - auto sampled_mips = test.sampled_image_rgba_mips; - auto sampled_format = test.sampled_image_format; - auto sampled_dwords_per_pixel = test.sampled_image_dwords_per_pixel; - if (!test.sampled_image_rgba_mips.empty()) { - sampled_format = vk::Format::eR32G32B32A32Sfloat; - sampled_dwords_per_pixel = 4; - } else if (!test.sampled_image_rgba.empty()) { - sampled_mips.push_back(test.sampled_image_rgba); - } - sampled_image = vulkan->CreateImageMips( - test.name, test.image_width, test.image_height, sampled_format, - vk::ImageUsageFlagBits::eSampled, sampled_mips, sampled_dwords_per_pixel, - vk::ImageLayout::eShaderReadOnlyOptimal, test.sampled_image_type, - test.sampled_image_view_type, test.sampled_image_layers, - test.sampled_image_view_base_layer, test.sampled_image_view_layers); - } - if (needs_storage_image) { - storage_image = vulkan->CreateImage2D( - test.name, test.image_width, test.image_height, test.storage_image_format, - vk::ImageUsageFlagBits::eStorage, test.storage_image_rgba, - test.storage_image_dwords_per_pixel, vk::ImageLayout::eGeneral); - storage_image_uint = - vulkan->CreateImage2D(test.name, test.image_width, test.image_height, - vk::Format::eR32Uint, vk::ImageUsageFlagBits::eStorage, - test.storage_image_r32ui, 1, vk::ImageLayout::eGeneral); - } - if (needs_sampler) { - sampler = vulkan->CreateNearestSampler(test.name); - } + using Kind = ShaderRecompiler::IR::DescriptorBindingKind; + auto Has = [&](Kind kind) { + return ShaderRecompiler::IR::FindBinding(compiled.program.bindings, kind) != + nullptr; + }; + VulkanHarness::Image sampled_image; + VulkanHarness::Image storage_image; + VulkanHarness::Image storage_image_uint; + VulkanHarness::Buffer gds_buffer; + vk::Sampler sampler = nullptr; + const bool needs_sampled_image = + Has(Kind::Sampled1D) || Has(Kind::Sampled1DArray) || + Has(Kind::Sampled2D) || Has(Kind::Sampled2DArray) || + Has(Kind::Sampled3D) || Has(Kind::SampledUint2D) || + Has(Kind::SampledUint1D) || Has(Kind::SampledUint1DArray) || + Has(Kind::SampledUint2DArray) || Has(Kind::SampledUint3D); + const bool needs_storage_image = + Has(Kind::Storage1D) || Has(Kind::Storage1DArray) || + Has(Kind::Storage2D) || Has(Kind::Storage2DArray) || + Has(Kind::Storage3D) || Has(Kind::StorageUint2D) || + Has(Kind::StorageUint1D) || Has(Kind::StorageUint1DArray) || + Has(Kind::StorageUint2DArray) || Has(Kind::StorageUint3D); + const bool needs_sampler = Has(Kind::Samplers); + const bool needs_gds = Has(Kind::Gds); + if (needs_gds) { + const auto gds_dwords = std::max( + {test.gds_initial.size(), test.expected_gds.size(), 1u}); + gds_buffer = + vulkan->CreateStorageBuffer(test.name, test.gds_initial, gds_dwords); + } + if (needs_sampled_image) { + auto sampled_mips = test.sampled_image_rgba_mips; + auto sampled_format = test.sampled_image_format; + auto sampled_dwords_per_pixel = test.sampled_image_dwords_per_pixel; + if (!test.sampled_image_rgba_mips.empty()) { + sampled_format = vk::Format::eR32G32B32A32Sfloat; + sampled_dwords_per_pixel = 4; + } else if (!test.sampled_image_rgba.empty()) { + sampled_mips.push_back(test.sampled_image_rgba); + } + sampled_image = vulkan->CreateImageMips( + test.name, test.image_width, test.image_height, sampled_format, + vk::ImageUsageFlagBits::eSampled, sampled_mips, + sampled_dwords_per_pixel, vk::ImageLayout::eShaderReadOnlyOptimal, + test.sampled_image_type, test.sampled_image_view_type, + test.sampled_image_layers, test.sampled_image_view_base_layer, + test.sampled_image_view_layers); + } + if (needs_storage_image) { + storage_image = vulkan->CreateImage2D( + test.name, test.image_width, test.image_height, + test.storage_image_format, vk::ImageUsageFlagBits::eStorage, + test.storage_image_rgba, test.storage_image_dwords_per_pixel, + vk::ImageLayout::eGeneral); + storage_image_uint = vulkan->CreateImage2D( + test.name, test.image_width, test.image_height, vk::Format::eR32Uint, + vk::ImageUsageFlagBits::eStorage, test.storage_image_r32ui, 1, + vk::ImageLayout::eGeneral); + } + if (needs_sampler) { + sampler = vulkan->CreateNearestSampler(test.name); + } - vulkan->Dispatch(test, compiled, buffer, needs_gds ? &gds_buffer : nullptr, - needs_sampled_image ? &sampled_image : nullptr, - needs_storage_image ? &storage_image : nullptr, - needs_storage_image ? &storage_image_uint : nullptr, sampler); - auto actual = vulkan->ReadBuffer(test.name, buffer, test.expected.size()); - if (!test.expected_gds.empty()) { - const auto gds_actual = vulkan->ReadBuffer(test.name, gds_buffer, test.expected_gds.size()); - CompareWords(test, "GDS readback", test.expected_gds, gds_actual); - } - if (!test.expected_storage_image_rgba.empty()) { - auto image_actual = vulkan->ReadImage(test.name, &storage_image); - image_actual.resize(test.expected_storage_image_rgba.size()); - CompareWords(test, "storage image readback", test.expected_storage_image_rgba, - image_actual); - } - if (!test.expected_storage_image_r32ui.empty()) { - auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint); - image_actual.resize(test.expected_storage_image_r32ui.size()); - CompareWords(test, "uint storage image readback", test.expected_storage_image_r32ui, - image_actual); - } - if (sampler != nullptr) { - vulkan->Device().destroySampler(sampler, nullptr); - } - vulkan->DestroyImage(&sampled_image); - vulkan->DestroyImage(&storage_image); - vulkan->DestroyImage(&storage_image_uint); - vulkan->DestroyBuffer(&gds_buffer); - vulkan->DestroyBuffer(&buffer); - CompareWords(test, "readback", test.expected, actual); - std::printf("[compute] %-32s ok\n", test.name); + vulkan->Dispatch(test, compiled, buffer, needs_gds ? &gds_buffer : nullptr, + needs_sampled_image ? &sampled_image : nullptr, + needs_storage_image ? &storage_image : nullptr, + needs_storage_image ? &storage_image_uint : nullptr, + sampler); + auto actual = vulkan->ReadBuffer(test.name, buffer, test.expected.size()); + if (!test.expected_gds.empty()) { + const auto gds_actual = + vulkan->ReadBuffer(test.name, gds_buffer, test.expected_gds.size()); + CompareWords(test, "GDS readback", test.expected_gds, gds_actual); + } + if (!test.expected_storage_image_rgba.empty()) { + auto image_actual = vulkan->ReadImage(test.name, &storage_image); + image_actual.resize(test.expected_storage_image_rgba.size()); + CompareWords(test, "storage image readback", + test.expected_storage_image_rgba, image_actual); + } + if (!test.expected_storage_image_r32ui.empty()) { + auto image_actual = vulkan->ReadImage(test.name, &storage_image_uint); + image_actual.resize(test.expected_storage_image_r32ui.size()); + CompareWords(test, "uint storage image readback", + test.expected_storage_image_r32ui, image_actual); + } + if (sampler != nullptr) { + vulkan->Device().destroySampler(sampler, nullptr); + } + vulkan->DestroyImage(&sampled_image); + vulkan->DestroyImage(&storage_image); + vulkan->DestroyImage(&storage_image_uint); + vulkan->DestroyBuffer(&gds_buffer); + vulkan->DestroyBuffer(&buffer); + CompareWords(test, "readback", test.expected, actual); + std::printf("[compute] %-32s ok\n", test.name); } -void RunGraphicsCase(VulkanHarness* vulkan, const GraphicsCase& test) { - auto compiled = CompileFragmentCase(test); - auto actual = vulkan->RenderFragment(test, compiled); - CompareGraphicsWords(test, actual); - std::printf("[graphics] %-31s ok\n", test.name); +void RunGraphicsCase(VulkanHarness *vulkan, const GraphicsCase &test) { + auto compiled = CompileFragmentCase(test); + auto actual = vulkan->RenderFragment(test, compiled); + CompareGraphicsWords(test, actual); + std::printf("[graphics] %-31s ok\n", test.name); } enum class CoverageClass { - Covered, - ControlOrMarker, - NeedsAluCase, - NeedsFloatCase, - NeedsMemoryCase, - NeedsImageCase, - NeedsGraphicsStageCase, + Covered, + ControlOrMarker, + NeedsAluCase, + NeedsFloatCase, + NeedsMemoryCase, + NeedsImageCase, + NeedsGraphicsStageCase, }; -bool IsCovered(const std::set& covered, ShaderOpcode opcode) { - return covered.find(opcode) != covered.end(); +bool IsCovered(const std::set &covered, ShaderOpcode opcode) { + return covered.find(opcode) != covered.end(); } -CoverageClass ClassifyOpcode(ShaderOpcode opcode, const std::set& covered) { - using ShaderRecompiler::Decoder::Opcode; +CoverageClass ClassifyOpcode(ShaderOpcode opcode, + const std::set &covered) { + using ShaderRecompiler::Decoder::Opcode; - if (IsCovered(covered, opcode)) { - return CoverageClass::Covered; - } + if (IsCovered(covered, opcode)) { + return CoverageClass::Covered; + } - switch (opcode) { - case Opcode::SGetpcB64: - case Opcode::SSetpcB64: - case Opcode::SNop: - case Opcode::SWaitcnt: - case Opcode::SBarrier: - case Opcode::SBranch: - case Opcode::SCbranchScc0: - case Opcode::SCbranchScc1: - case Opcode::SCbranchVccz: - case Opcode::SCbranchVccnz: - case Opcode::SCbranchExecz: - case Opcode::SCbranchExecnz: - case Opcode::SSendmsg: - case Opcode::SSetregB32: - case Opcode::SSleep: - case Opcode::STtraceData: - case Opcode::SInstPrefetch: - case Opcode::SEndpgm: return CoverageClass::ControlOrMarker; + switch (opcode) { + case Opcode::SGetpcB64: + case Opcode::SSetpcB64: + case Opcode::SNop: + case Opcode::SWaitcnt: + case Opcode::SBarrier: + case Opcode::SBranch: + case Opcode::SCbranchScc0: + case Opcode::SCbranchScc1: + case Opcode::SCbranchVccz: + case Opcode::SCbranchVccnz: + case Opcode::SCbranchExecz: + case Opcode::SCbranchExecnz: + case Opcode::SSendmsg: + case Opcode::SSetregB32: + case Opcode::SSleep: + case Opcode::STtraceData: + case Opcode::SInstPrefetch: + case Opcode::SEndpgm: + return CoverageClass::ControlOrMarker; - case Opcode::VAddF32: - case Opcode::VSubF32: - case Opcode::VSubrevF32: - case Opcode::VMulF32: - case Opcode::VMacF32: - case Opcode::VMadmkF32: - case Opcode::VMadakF32: - case Opcode::VMinF32: - case Opcode::VMaxF32: - case Opcode::VMadF32: - case Opcode::VFmaF32: - case Opcode::VMin3F32: - case Opcode::VMax3F32: - case Opcode::VMed3F32: - case Opcode::VDot2cF32F16: - case Opcode::VCvtF32I32: - case Opcode::VCvtF32U32: - case Opcode::VCvtU32F32: - case Opcode::VCvtI32F32: - case Opcode::VCvtF16F32: - case Opcode::VCvtF32F16: - case Opcode::VCvtU16F16: - case Opcode::VCvtRpiI32F32: - case Opcode::VCvtFlrI32F32: - case Opcode::VCvtOffF32I4: - case Opcode::VCvtF32Ubyte0: - case Opcode::VCvtF32Ubyte1: - case Opcode::VCvtF32Ubyte2: - case Opcode::VCvtF32Ubyte3: - case Opcode::VRcpF32: - case Opcode::VFractF32: - case Opcode::VTruncF32: - case Opcode::VCeilF32: - case Opcode::VRndneF32: - case Opcode::VFloorF32: - case Opcode::VExpF32: - case Opcode::VLogF32: - case Opcode::VRsqF32: - case Opcode::VSqrtF32: - case Opcode::VSinF32: - case Opcode::VCosF32: - case Opcode::VCubeidF32: - case Opcode::VCubescF32: - case Opcode::VCubetcF32: - case Opcode::VCubemaF32: - case Opcode::VLdexpF32: - case Opcode::VCvtPkU8F32: - case Opcode::VCvtPknormI16F32: - case Opcode::VCvtPknormU16F32: - case Opcode::VCvtPkrtzF16F32: - case Opcode::VPkAddF16: - case Opcode::VPkMulF16: - case Opcode::VPkMinF16: - case Opcode::VPkMaxF16: - case Opcode::VPkFmaF16: - case Opcode::VAddF16: - case Opcode::VSubF16: - case Opcode::VSubrevF16: - case Opcode::VMulF16: - case Opcode::VMaxF16: - case Opcode::VMinF16: - case Opcode::VMin3F16: - case Opcode::VMax3F16: - case Opcode::VMed3F16: - case Opcode::VRcpF16: - case Opcode::VRsqF16: - case Opcode::VLogF16: - case Opcode::VExpF16: - case Opcode::VMadMixloF16: - case Opcode::VMadMixhiF16: - case Opcode::DsMinF32: - case Opcode::DsMaxF32: - case Opcode::VCmpFF32: - case Opcode::VCmpLtF32: - case Opcode::VCmpEqF32: - case Opcode::VCmpLeF32: - case Opcode::VCmpGtF32: - case Opcode::VCmpLgF32: - case Opcode::VCmpGeF32: - case Opcode::VCmpOF32: - case Opcode::VCmpUF32: - case Opcode::VCmpNgeF32: - case Opcode::VCmpNlgF32: - case Opcode::VCmpNgtF32: - case Opcode::VCmpNleF32: - case Opcode::VCmpNeqF32: - case Opcode::VCmpNltF32: - case Opcode::VCmpTruF32: - case Opcode::VCmpxLtF32: - case Opcode::VCmpxEqF32: - case Opcode::VCmpxLeF32: - case Opcode::VCmpxGtF32: - case Opcode::VCmpxLgF32: - case Opcode::VCmpxGeF32: - case Opcode::VCmpxNgeF32: - case Opcode::VCmpxNlgF32: - case Opcode::VCmpxNgtF32: - case Opcode::VCmpxNleF32: - case Opcode::VCmpxNeqF32: - case Opcode::VCmpxNltF32: - case Opcode::VCmpClassF32: - case Opcode::VCmpLtF16: - case Opcode::VCmpEqF16: - case Opcode::VCmpLeF16: - case Opcode::VCmpGtF16: - case Opcode::VCmpLgF16: - case Opcode::VCmpGeF16: - case Opcode::VCmpNeqF16: - case Opcode::VCmpxLtF16: - case Opcode::VCmpxEqF16: - case Opcode::VCmpxLeF16: - case Opcode::VCmpxGtF16: - case Opcode::VCmpxGeF16: - case Opcode::VCmpxNeqF16: - case Opcode::VCmpxNltF16: return CoverageClass::NeedsFloatCase; + case Opcode::VAddF32: + case Opcode::VSubF32: + case Opcode::VSubrevF32: + case Opcode::VMulF32: + case Opcode::VMacF32: + case Opcode::VMadmkF32: + case Opcode::VMadakF32: + case Opcode::VMinF32: + case Opcode::VMaxF32: + case Opcode::VMadF32: + case Opcode::VFmaF32: + case Opcode::VMin3F32: + case Opcode::VMax3F32: + case Opcode::VMed3F32: + case Opcode::VDot2cF32F16: + case Opcode::VCvtF32I32: + case Opcode::VCvtF32U32: + case Opcode::VCvtU32F32: + case Opcode::VCvtI32F32: + case Opcode::VCvtF16F32: + case Opcode::VCvtF32F16: + case Opcode::VCvtU16F16: + case Opcode::VCvtRpiI32F32: + case Opcode::VCvtFlrI32F32: + case Opcode::VCvtOffF32I4: + case Opcode::VCvtF32Ubyte0: + case Opcode::VCvtF32Ubyte1: + case Opcode::VCvtF32Ubyte2: + case Opcode::VCvtF32Ubyte3: + case Opcode::VRcpF32: + case Opcode::VFractF32: + case Opcode::VTruncF32: + case Opcode::VCeilF32: + case Opcode::VRndneF32: + case Opcode::VFloorF32: + case Opcode::VExpF32: + case Opcode::VLogF32: + case Opcode::VRsqF32: + case Opcode::VSqrtF32: + case Opcode::VSinF32: + case Opcode::VCosF32: + case Opcode::VCubeidF32: + case Opcode::VCubescF32: + case Opcode::VCubetcF32: + case Opcode::VCubemaF32: + case Opcode::VLdexpF32: + case Opcode::VCvtPkU8F32: + case Opcode::VCvtPknormI16F32: + case Opcode::VCvtPknormU16F32: + case Opcode::VCvtPkrtzF16F32: + case Opcode::VPkAddF16: + case Opcode::VPkMulF16: + case Opcode::VPkMinF16: + case Opcode::VPkMaxF16: + case Opcode::VPkFmaF16: + case Opcode::VAddF16: + case Opcode::VSubF16: + case Opcode::VSubrevF16: + case Opcode::VMulF16: + case Opcode::VMaxF16: + case Opcode::VMinF16: + case Opcode::VMin3F16: + case Opcode::VMax3F16: + case Opcode::VMed3F16: + case Opcode::VRcpF16: + case Opcode::VRsqF16: + case Opcode::VLogF16: + case Opcode::VExpF16: + case Opcode::VMadMixloF16: + case Opcode::VMadMixhiF16: + case Opcode::DsMinF32: + case Opcode::DsMaxF32: + case Opcode::VCmpFF32: + case Opcode::VCmpLtF32: + case Opcode::VCmpEqF32: + case Opcode::VCmpLeF32: + case Opcode::VCmpGtF32: + case Opcode::VCmpLgF32: + case Opcode::VCmpGeF32: + case Opcode::VCmpOF32: + case Opcode::VCmpUF32: + case Opcode::VCmpNgeF32: + case Opcode::VCmpNlgF32: + case Opcode::VCmpNgtF32: + case Opcode::VCmpNleF32: + case Opcode::VCmpNeqF32: + case Opcode::VCmpNltF32: + case Opcode::VCmpTruF32: + case Opcode::VCmpxLtF32: + case Opcode::VCmpxEqF32: + case Opcode::VCmpxLeF32: + case Opcode::VCmpxGtF32: + case Opcode::VCmpxLgF32: + case Opcode::VCmpxGeF32: + case Opcode::VCmpxNgeF32: + case Opcode::VCmpxNlgF32: + case Opcode::VCmpxNgtF32: + case Opcode::VCmpxNleF32: + case Opcode::VCmpxNeqF32: + case Opcode::VCmpxNltF32: + case Opcode::VCmpClassF32: + case Opcode::VCmpLtF16: + case Opcode::VCmpEqF16: + case Opcode::VCmpLeF16: + case Opcode::VCmpGtF16: + case Opcode::VCmpLgF16: + case Opcode::VCmpGeF16: + case Opcode::VCmpNeqF16: + case Opcode::VCmpxLtF16: + case Opcode::VCmpxEqF16: + case Opcode::VCmpxLeF16: + case Opcode::VCmpxGtF16: + case Opcode::VCmpxGeF16: + case Opcode::VCmpxNeqF16: + case Opcode::VCmpxNltF16: + return CoverageClass::NeedsFloatCase; - case Opcode::SLoadDword: - case Opcode::SLoadDwordx2: - case Opcode::SLoadDwordx4: - case Opcode::SLoadDwordx8: - case Opcode::SLoadDwordx16: - case Opcode::SBufferLoadDword: - case Opcode::SBufferLoadDwordx2: - case Opcode::SBufferLoadDwordx4: - case Opcode::SBufferLoadDwordx8: - case Opcode::SBufferLoadDwordx16: - case Opcode::BufferLoadFormatX: - case Opcode::BufferLoadFormatXy: - case Opcode::BufferLoadFormatXyz: - case Opcode::BufferLoadFormatXyzw: - case Opcode::BufferStoreFormatX: - case Opcode::BufferStoreFormatXy: - case Opcode::BufferStoreFormatXyz: - case Opcode::BufferStoreFormatXyzw: - case Opcode::BufferLoadUbyte: - case Opcode::BufferLoadSbyte: - case Opcode::BufferLoadUshort: - case Opcode::BufferLoadSshort: - case Opcode::BufferLoadDwordx2: - case Opcode::BufferLoadDwordx3: - case Opcode::BufferLoadDwordx4: - case Opcode::BufferStoreByte: - case Opcode::BufferStoreShort: - case Opcode::BufferStoreDwordx2: - case Opcode::BufferStoreDwordx3: - case Opcode::BufferStoreDwordx4: - case Opcode::TBufferLoadFormatX: - case Opcode::TBufferLoadFormatXy: - case Opcode::TBufferLoadFormatXyz: - case Opcode::TBufferLoadFormatXyzw: - case Opcode::TBufferStoreFormatX: - case Opcode::TBufferStoreFormatXy: - case Opcode::TBufferStoreFormatXyz: - case Opcode::TBufferStoreFormatXyzw: - case Opcode::BufferAtomicSwap: - case Opcode::BufferAtomicAdd: - case Opcode::BufferAtomicSub: - case Opcode::BufferAtomicSMin: - case Opcode::BufferAtomicUMin: - case Opcode::BufferAtomicSMax: - case Opcode::BufferAtomicUMax: - case Opcode::BufferAtomicAnd: - case Opcode::BufferAtomicOr: - case Opcode::BufferAtomicXor: - case Opcode::BufferAtomicFMin: - case Opcode::BufferAtomicFMax: - case Opcode::FlatLoadUbyte: - case Opcode::FlatLoadSbyte: - case Opcode::FlatLoadUshort: - case Opcode::FlatLoadSshort: - case Opcode::FlatLoadDword: - case Opcode::FlatLoadDwordx2: - case Opcode::FlatLoadDwordx3: - case Opcode::FlatLoadDwordx4: - case Opcode::FlatStoreByte: - case Opcode::FlatStoreShort: - case Opcode::FlatStoreDword: - case Opcode::FlatStoreDwordx2: - case Opcode::FlatStoreDwordx3: - case Opcode::FlatStoreDwordx4: - case Opcode::DsAddU32: - case Opcode::DsAddRtnU32: - case Opcode::DsSubU32: - case Opcode::DsSubRtnU32: - case Opcode::DsMinI32: - case Opcode::DsMinRtnI32: - case Opcode::DsMaxI32: - case Opcode::DsMaxRtnI32: - case Opcode::DsMinU32: - case Opcode::DsMinRtnU32: - case Opcode::DsMaxU32: - case Opcode::DsMaxRtnU32: - case Opcode::DsAndB32: - case Opcode::DsAndRtnB32: - case Opcode::DsOrB32: - case Opcode::DsOrRtnB32: - case Opcode::DsXorB32: - case Opcode::DsXorRtnB32: - case Opcode::DsWrxchgRtnB32: - case Opcode::DsSwizzleB32: - case Opcode::DsReadSbyte: - case Opcode::DsReadUbyte: - case Opcode::DsReadSshort: - case Opcode::DsReadUshort: - case Opcode::DsRead2B32: - case Opcode::DsReadB32: - case Opcode::DsReadB64: - case Opcode::DsRead2B64: - case Opcode::DsReadB96: - case Opcode::DsReadB128: - case Opcode::DsWriteByte: - case Opcode::DsWriteShort: - case Opcode::DsWrite2B32: - case Opcode::DsWrite2St64B32: - case Opcode::DsWriteB32: - case Opcode::DsWriteB64: - case Opcode::DsWriteB96: - case Opcode::DsWriteB128: - case Opcode::DsWriteAddtidB32: - case Opcode::DsReadAddtidB32: return CoverageClass::NeedsMemoryCase; + case Opcode::SLoadDword: + case Opcode::SLoadDwordx2: + case Opcode::SLoadDwordx4: + case Opcode::SLoadDwordx8: + case Opcode::SLoadDwordx16: + case Opcode::SBufferLoadDword: + case Opcode::SBufferLoadDwordx2: + case Opcode::SBufferLoadDwordx4: + case Opcode::SBufferLoadDwordx8: + case Opcode::SBufferLoadDwordx16: + case Opcode::BufferLoadFormatX: + case Opcode::BufferLoadFormatXy: + case Opcode::BufferLoadFormatXyz: + case Opcode::BufferLoadFormatXyzw: + case Opcode::BufferStoreFormatX: + case Opcode::BufferStoreFormatXy: + case Opcode::BufferStoreFormatXyz: + case Opcode::BufferStoreFormatXyzw: + case Opcode::BufferLoadUbyte: + case Opcode::BufferLoadSbyte: + case Opcode::BufferLoadUshort: + case Opcode::BufferLoadSshort: + case Opcode::BufferLoadDwordx2: + case Opcode::BufferLoadDwordx3: + case Opcode::BufferLoadDwordx4: + case Opcode::BufferStoreByte: + case Opcode::BufferStoreShort: + case Opcode::BufferStoreDwordx2: + case Opcode::BufferStoreDwordx3: + case Opcode::BufferStoreDwordx4: + case Opcode::TBufferLoadFormatX: + case Opcode::TBufferLoadFormatXy: + case Opcode::TBufferLoadFormatXyz: + case Opcode::TBufferLoadFormatXyzw: + case Opcode::TBufferStoreFormatX: + case Opcode::TBufferStoreFormatXy: + case Opcode::TBufferStoreFormatXyz: + case Opcode::TBufferStoreFormatXyzw: + case Opcode::BufferAtomicSwap: + case Opcode::BufferAtomicAdd: + case Opcode::BufferAtomicSub: + case Opcode::BufferAtomicSMin: + case Opcode::BufferAtomicUMin: + case Opcode::BufferAtomicSMax: + case Opcode::BufferAtomicUMax: + case Opcode::BufferAtomicAnd: + case Opcode::BufferAtomicOr: + case Opcode::BufferAtomicXor: + case Opcode::BufferAtomicFMin: + case Opcode::BufferAtomicFMax: + case Opcode::FlatLoadUbyte: + case Opcode::FlatLoadSbyte: + case Opcode::FlatLoadUshort: + case Opcode::FlatLoadSshort: + case Opcode::FlatLoadDword: + case Opcode::FlatLoadDwordx2: + case Opcode::FlatLoadDwordx3: + case Opcode::FlatLoadDwordx4: + case Opcode::FlatStoreByte: + case Opcode::FlatStoreShort: + case Opcode::FlatStoreDword: + case Opcode::FlatStoreDwordx2: + case Opcode::FlatStoreDwordx3: + case Opcode::FlatStoreDwordx4: + case Opcode::DsAddU32: + case Opcode::DsAddRtnU32: + case Opcode::DsSubU32: + case Opcode::DsSubRtnU32: + case Opcode::DsMinI32: + case Opcode::DsMinRtnI32: + case Opcode::DsMaxI32: + case Opcode::DsMaxRtnI32: + case Opcode::DsMinU32: + case Opcode::DsMinRtnU32: + case Opcode::DsMaxU32: + case Opcode::DsMaxRtnU32: + case Opcode::DsAndB32: + case Opcode::DsAndRtnB32: + case Opcode::DsOrB32: + case Opcode::DsOrRtnB32: + case Opcode::DsXorB32: + case Opcode::DsXorRtnB32: + case Opcode::DsWrxchgRtnB32: + case Opcode::DsSwizzleB32: + case Opcode::DsReadSbyte: + case Opcode::DsReadUbyte: + case Opcode::DsReadSshort: + case Opcode::DsReadUshort: + case Opcode::DsRead2B32: + case Opcode::DsReadB32: + case Opcode::DsReadB64: + case Opcode::DsRead2B64: + case Opcode::DsReadB96: + case Opcode::DsReadB128: + case Opcode::DsWriteByte: + case Opcode::DsWriteShort: + case Opcode::DsWrite2B32: + case Opcode::DsWrite2St64B32: + case Opcode::DsWriteB32: + case Opcode::DsWriteB64: + case Opcode::DsWriteB96: + case Opcode::DsWriteB128: + case Opcode::DsWriteAddtidB32: + case Opcode::DsReadAddtidB32: + return CoverageClass::NeedsMemoryCase; - case Opcode::ImageGetResinfo: - case Opcode::ImageGetLod: - case Opcode::ImageLoad: - case Opcode::ImageLoadMip: - case Opcode::ImageStore: - case Opcode::ImageStoreMip: - case Opcode::ImageAtomicAdd: - case Opcode::ImageAtomicUMin: - case Opcode::ImageAtomicAnd: - case Opcode::ImageAtomicOr: - case Opcode::ImageAtomicXor: - case Opcode::ImageSample: - case Opcode::ImageGather4Lz: - case Opcode::ImageGather4C: - case Opcode::ImageGather4CLz: - case Opcode::ImageGather4LzO: - case Opcode::ImageGather4CO: - case Opcode::ImageGather4CLzO: return CoverageClass::NeedsImageCase; + case Opcode::ImageGetResinfo: + case Opcode::ImageGetLod: + case Opcode::ImageLoad: + case Opcode::ImageLoadMip: + case Opcode::ImageStore: + case Opcode::ImageStoreMip: + case Opcode::ImageAtomicAdd: + case Opcode::ImageAtomicUMin: + case Opcode::ImageAtomicAnd: + case Opcode::ImageAtomicOr: + case Opcode::ImageAtomicXor: + case Opcode::ImageSample: + case Opcode::ImageGather4Lz: + case Opcode::ImageGather4C: + case Opcode::ImageGather4CLz: + case Opcode::ImageGather4LzO: + case Opcode::ImageGather4CO: + case Opcode::ImageGather4CLzO: + return CoverageClass::NeedsImageCase; - case Opcode::VInterpP1F32: - case Opcode::VInterpP2F32: - case Opcode::VInterpMovF32: - case Opcode::Exp: return CoverageClass::NeedsGraphicsStageCase; + case Opcode::VInterpP1F32: + case Opcode::VInterpP2F32: + case Opcode::VInterpMovF32: + case Opcode::Exp: + return CoverageClass::NeedsGraphicsStageCase; - default: return CoverageClass::NeedsAluCase; - } + default: + return CoverageClass::NeedsAluCase; + } } -const char* CoverageClassName(CoverageClass status) { - switch (status) { - case CoverageClass::Covered: return "covered"; - case CoverageClass::ControlOrMarker: return "control"; - case CoverageClass::NeedsAluCase: return "alu"; - case CoverageClass::NeedsFloatCase: return "float"; - case CoverageClass::NeedsMemoryCase: return "memory"; - case CoverageClass::NeedsImageCase: return "image"; - case CoverageClass::NeedsGraphicsStageCase: return "graphics"; - default: return "unknown"; - } +const char *CoverageClassName(CoverageClass status) { + switch (status) { + case CoverageClass::Covered: + return "covered"; + case CoverageClass::ControlOrMarker: + return "control"; + case CoverageClass::NeedsAluCase: + return "alu"; + case CoverageClass::NeedsFloatCase: + return "float"; + case CoverageClass::NeedsMemoryCase: + return "memory"; + case CoverageClass::NeedsImageCase: + return "image"; + case CoverageClass::NeedsGraphicsStageCase: + return "graphics"; + default: + return "unknown"; + } } -void PrintPendingOpcodes(CoverageClass status, const std::vector& opcodes) { - if (opcodes.empty()) { - return; - } +void PrintPendingOpcodes(CoverageClass status, + const std::vector &opcodes) { + if (opcodes.empty()) { + return; + } - std::printf("[coverage] pending_%s:", CoverageClassName(status)); - for (auto opcode: opcodes) { - const auto name = ShaderRecompiler::Decoder::OpcodeToString(opcode); - std::printf(" %s", name.c_str()); - } - std::printf("\n"); + std::printf("[coverage] pending_%s:", CoverageClassName(status)); + for (auto opcode : opcodes) { + const auto name = ShaderRecompiler::Decoder::OpcodeToString(opcode); + std::printf(" %s", name.c_str()); + } + std::printf("\n"); } -void CheckOpcodeCoverage(const std::vector& tests, - const std::vector& graphics_tests) { - using ShaderRecompiler::Decoder::Opcode; +void CheckOpcodeCoverage(const std::vector &tests, + const std::vector &graphics_tests) { + using ShaderRecompiler::Decoder::Opcode; - std::set covered; - for (const auto& test: tests) { - for (auto opcode: test.opcodes) { - covered.insert(opcode); - } - } - for (const auto& test: graphics_tests) { - for (auto opcode: test.opcodes) { - covered.insert(opcode); - } - } + std::set covered; + for (const auto &test : tests) { + for (auto opcode : test.opcodes) { + covered.insert(opcode); + } + } + for (const auto &test : graphics_tests) { + for (auto opcode : test.opcodes) { + covered.insert(opcode); + } + } - uint32_t counts[7] = {}; - std::vector pending[7]; - for (auto value = static_cast(Opcode::SMovB32); value <= static_cast(Opcode::Exp); - value++) { - const auto opcode = static_cast(value); - const auto status = ClassifyOpcode(opcode, covered); - counts[static_cast(status)]++; - if (status != CoverageClass::Covered && status != CoverageClass::ControlOrMarker) { - pending[static_cast(status)].push_back(opcode); - } - } + uint32_t counts[7] = {}; + std::vector pending[7]; + for (auto value = static_cast(Opcode::SMovB32); + value <= static_cast(Opcode::Exp); value++) { + const auto opcode = static_cast(value); + const auto status = ClassifyOpcode(opcode, covered); + counts[static_cast(status)]++; + if (status != CoverageClass::Covered && + status != CoverageClass::ControlOrMarker) { + pending[static_cast(status)].push_back(opcode); + } + } - std::printf("[coverage] decoder opcodes: covered=%u control=%u alu_pending=%u " - "float_pending=%u memory_pending=%u image_pending=%u " - "graphics_pending=%u\n", - counts[static_cast(CoverageClass::Covered)], - counts[static_cast(CoverageClass::ControlOrMarker)], - counts[static_cast(CoverageClass::NeedsAluCase)], - counts[static_cast(CoverageClass::NeedsFloatCase)], - counts[static_cast(CoverageClass::NeedsMemoryCase)], - counts[static_cast(CoverageClass::NeedsImageCase)], - counts[static_cast(CoverageClass::NeedsGraphicsStageCase)]); - PrintPendingOpcodes(CoverageClass::NeedsAluCase, - pending[static_cast(CoverageClass::NeedsAluCase)]); - PrintPendingOpcodes(CoverageClass::NeedsFloatCase, - pending[static_cast(CoverageClass::NeedsFloatCase)]); - PrintPendingOpcodes(CoverageClass::NeedsMemoryCase, - pending[static_cast(CoverageClass::NeedsMemoryCase)]); - PrintPendingOpcodes(CoverageClass::NeedsImageCase, - pending[static_cast(CoverageClass::NeedsImageCase)]); - PrintPendingOpcodes(CoverageClass::NeedsGraphicsStageCase, - pending[static_cast(CoverageClass::NeedsGraphicsStageCase)]); + std::printf( + "[coverage] decoder opcodes: covered=%u control=%u alu_pending=%u " + "float_pending=%u memory_pending=%u image_pending=%u " + "graphics_pending=%u\n", + counts[static_cast(CoverageClass::Covered)], + counts[static_cast(CoverageClass::ControlOrMarker)], + counts[static_cast(CoverageClass::NeedsAluCase)], + counts[static_cast(CoverageClass::NeedsFloatCase)], + counts[static_cast(CoverageClass::NeedsMemoryCase)], + counts[static_cast(CoverageClass::NeedsImageCase)], + counts[static_cast(CoverageClass::NeedsGraphicsStageCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsAluCase, + pending[static_cast(CoverageClass::NeedsAluCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsFloatCase, + pending[static_cast(CoverageClass::NeedsFloatCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsMemoryCase, + pending[static_cast(CoverageClass::NeedsMemoryCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsImageCase, + pending[static_cast(CoverageClass::NeedsImageCase)]); + PrintPendingOpcodes( + CoverageClass::NeedsGraphicsStageCase, + pending[static_cast(CoverageClass::NeedsGraphicsStageCase)]); } TestCase IntegerAddSubMul() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeSMovB32(0, InlineU32(7)), - EncodeSMovB32(1, InlineU32(3)), - EncodeSop2(0x00, 2, 0, 1), - EncodeSop2(0x01, 3, 2, InlineU32(1)), - EncodeSop2(0x26, 4, 3, InlineU32(2)), - EncodeVop1(0x01, 0, 4), - }; - AppendBufferStoreDword(&code, 0, 30); - AppendEnd(&code); - return {"IntegerAddSubMul", - code, - {}, - {18}, - {O::SMovB32, O::SAddU32, O::SSubU32, O::SMulI32, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + std::vector code = { + EncodeSMovB32(0, InlineU32(7)), + EncodeSMovB32(1, InlineU32(3)), + EncodeSop2(0x00, 2, 0, 1), + EncodeSop2(0x01, 3, 2, InlineU32(1)), + EncodeSop2(0x26, 4, 3, InlineU32(2)), + EncodeVop1(0x01, 0, 4), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"IntegerAddSubMul", + code, + {}, + {18}, + {O::SMovB32, O::SAddU32, O::SSubU32, O::SMulI32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase BitwiseOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeSMovB32(0, InlineU32(60)), EncodeSMovB32(1, InlineU32(15)), EncodeSop2(0x0e, 2, 0, 1), - EncodeSop2(0x10, 3, 0, 1), EncodeSop2(0x12, 4, 3, 2), EncodeVop1(0x37, 0, 4), - }; - AppendBufferStoreDword(&code, 0, 30); - AppendEnd(&code); - return {"BitwiseAndOrXorNot", - code, - {}, - {~0x33u}, - {O::SMovB32, O::SAndB32, O::SOrB32, O::SXorB32, O::VNotB32, O::BufferStoreDword, - O::SEndpgm}}; + std::vector code = { + EncodeSMovB32(0, InlineU32(60)), EncodeSMovB32(1, InlineU32(15)), + EncodeSop2(0x0e, 2, 0, 1), EncodeSop2(0x10, 3, 0, 1), + EncodeSop2(0x12, 4, 3, 2), EncodeVop1(0x37, 0, 4), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"BitwiseAndOrXorNot", + code, + {}, + {~0x33u}, + {O::SMovB32, O::SAndB32, O::SOrB32, O::SXorB32, O::VNotB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase Shifts() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeSMovB32(0, InlineU32(3)), - EncodeSop2(0x1e, 1, 0, InlineU32(2)), - EncodeSop2(0x20, 2, 1, InlineU32(1)), - EncodeVop1(0x01, 0, 2), - }; - AppendBufferStoreDword(&code, 0, 30); - AppendEnd(&code); - return {"Shifts", - code, - {}, - {6}, - {O::SMovB32, O::SLshlB32, O::SLshrB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + std::vector code = { + EncodeSMovB32(0, InlineU32(3)), + EncodeSop2(0x1e, 1, 0, InlineU32(2)), + EncodeSop2(0x20, 2, 1, InlineU32(1)), + EncodeVop1(0x01, 0, 2), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"Shifts", + code, + {}, + {6}, + {O::SMovB32, O::SLshlB32, O::SLshrB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ExactPushConstantExtent() { - using O = ShaderOpcode; + using O = ShaderOpcode; - constexpr std::array values {0x10203040u, 0x21314151u, 0x32425262u, 0x43536373u, - 0x54647484u, 0x65758595u, 0x768696a6u, 0x8797a7b7u, - 0x98a8b8c8u, 0xa9b9c9d9u}; - std::vector code; - for (u32 i = 0; i < values.size(); i++) { - AppendStoreSgpr(&code, 4u + i, i); - } - AppendEnd(&code); + constexpr std::array values{ + 0x10203040u, 0x21314151u, 0x32425262u, 0x43536373u, 0x54647484u, + 0x65758595u, 0x768696a6u, 0x8797a7b7u, 0x98a8b8c8u, 0xa9b9c9d9u}; + std::vector code; + for (u32 i = 0; i < values.size(); i++) { + AppendStoreSgpr(&code, 4u + i, i); + } + AppendEnd(&code); - TestCase test; - test.name = "ExactPushConstantExtent"; - test.code = std::move(code); - test.initial.resize(values.size()); - test.expected.assign(values.begin(), values.end()); - test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, values.size() * sizeof(u32)); - std::copy(values.begin(), values.end(), test.user_data.begin() + 4); - test.has_user_data = true; - return test; + TestCase test; + test.name = "ExactPushConstantExtent"; + test.code = std::move(code); + test.initial.resize(values.size()); + test.expected.assign(values.begin(), values.end()); + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.user_data = + MakeStructuredStorageBufferData(0, values.size() * sizeof(u32)); + std::copy(values.begin(), values.end(), test.user_data.begin() + 4); + test.has_user_data = true; + return test; } TestCase ScalarShiftCountsMaskLowBits() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(1))); - AppendSMovLiteral(&code, 1, 0x80000000u); - code.push_back(EncodeSMovB32(2, InlineU32(32))); - code.push_back(EncodeSMovB32(3, InlineU32(33))); - code.push_back(EncodeSop2(0x1e, 10, 0, 2)); - code.push_back(EncodeSop2(0x1e, 11, 0, 3)); - code.push_back(EncodeSop2(0x20, 12, 1, 2)); - code.push_back(EncodeSop2(0x20, 13, 1, 3)); - code.push_back(EncodeSop2(0x22, 14, 1, 2)); - code.push_back(EncodeSop2(0x22, 15, 1, 3)); + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(1))); + AppendSMovLiteral(&code, 1, 0x80000000u); + code.push_back(EncodeSMovB32(2, InlineU32(32))); + code.push_back(EncodeSMovB32(3, InlineU32(33))); + code.push_back(EncodeSop2(0x1e, 10, 0, 2)); + code.push_back(EncodeSop2(0x1e, 11, 0, 3)); + code.push_back(EncodeSop2(0x20, 12, 1, 2)); + code.push_back(EncodeSop2(0x20, 13, 1, 3)); + code.push_back(EncodeSop2(0x22, 14, 1, 2)); + code.push_back(EncodeSop2(0x22, 15, 1, 3)); - for (u32 i = 0; i < 6u; i++) { - AppendStoreSgpr(&code, 10u + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 6u; i++) { + AppendStoreSgpr(&code, 10u + i, i); + } + AppendEnd(&code); - return {"ScalarShiftCountsMaskLowBits", - code, - {}, - {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, - {O::SMovB32, O::SLshlB32, O::SLshrB32, O::SAshrI32, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarShiftCountsMaskLowBits", + code, + {}, + {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, + {O::SMovB32, O::SLshlB32, O::SLshrB32, O::SAshrI32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase Rdna2ScalarOpcodes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(2, InlineU32(7))); - code.push_back(EncodeSMovB32(106, InlineU32(16))); - code.push_back(EncodeSop1(0x1d, 106, InlineU32(0))); - AppendStoreSgpr(&code, 106, 0); - code.push_back(EncodeSopk(0x13, 106, 0x1019u)); - AppendStoreSgpr(&code, 106, 1); - code.push_back(EncodeSop2(0x02, 106, 2, 239u)); - AppendStoreSgpr(&code, 106, 2); - code.push_back(EncodeSopp(0x0e, 0)); - code.push_back(EncodeSop2(0x02, 106, 239u, 2)); - AppendStoreSgpr(&code, 106, 3); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSMovB32(2, InlineU32(7))); + code.push_back(EncodeSMovB32(106, InlineU32(16))); + code.push_back(EncodeSop1(0x1d, 106, InlineU32(0))); + AppendStoreSgpr(&code, 106, 0); + code.push_back(EncodeSopk(0x13, 106, 0x1019u)); + AppendStoreSgpr(&code, 106, 1); + code.push_back(EncodeSop2(0x02, 106, 2, 239u)); + AppendStoreSgpr(&code, 106, 2); + code.push_back(EncodeSopp(0x0e, 0)); + code.push_back(EncodeSop2(0x02, 106, 239u, 2)); + AppendStoreSgpr(&code, 106, 3); + AppendEnd(&code); - return {"Rdna2ScalarOpcodes", - code, - {}, - {0x11u, 0x11u, 7u, 7u}, - {O::SMovB32, O::SBitset1B32, O::SSetregB32, O::SAddI32, O::SSleep, O::VMovB32, - O::BufferStoreDword, O::SEndpgm}}; + return {"Rdna2ScalarOpcodes", + code, + {}, + {0x11u, 0x11u, 7u, 7u}, + {O::SMovB32, O::SBitset1B32, O::SSetregB32, O::SAddI32, O::SSleep, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarExtendedArithmetic() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 0xffffffffu); - code.push_back(EncodeSMovB32(1, InlineU32(1))); - code.push_back(EncodeSop2(0x00, 2, 0, 1)); - code.push_back(EncodeSMovB32(3, InlineU32(5))); - code.push_back(EncodeSMovB32(4, InlineU32(6))); - code.push_back(EncodeSop2(0x04, 5, 3, 4)); - code.push_back(EncodeSop2(0x02, 6, InlineU32(7), InlineU32(8))); - code.push_back(EncodeSop2(0x03, 7, InlineU32(7), InlineU32(9))); - AppendSMovLiteral(&code, 8, 0xfffffffbu); - code.push_back(EncodeSMovB32(9, InlineU32(3))); - code.push_back(EncodeSop2(0x06, 10, 8, 9)); - code.push_back(EncodeSop2(0x08, 11, 8, 9)); - AppendSMovLiteral(&code, 12, 0xfffffffeu); - code.push_back(EncodeSMovB32(13, InlineU32(3))); - code.push_back(EncodeSop2(0x07, 14, 12, 13)); - code.push_back(EncodeSop2(0x09, 15, 12, 13)); - code.push_back(EncodeSMovB32(16, 8)); - code.push_back(EncodeSop1(0x34, 16, 16)); - code.push_back(EncodeSopk(0x00, 17, 0xfff5u)); - code.push_back(EncodeSopk(0x10, 17, 0xfffdu)); + std::vector code; + AppendSMovLiteral(&code, 0, 0xffffffffu); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + code.push_back(EncodeSop2(0x00, 2, 0, 1)); + code.push_back(EncodeSMovB32(3, InlineU32(5))); + code.push_back(EncodeSMovB32(4, InlineU32(6))); + code.push_back(EncodeSop2(0x04, 5, 3, 4)); + code.push_back(EncodeSop2(0x02, 6, InlineU32(7), InlineU32(8))); + code.push_back(EncodeSop2(0x03, 7, InlineU32(7), InlineU32(9))); + AppendSMovLiteral(&code, 8, 0xfffffffbu); + code.push_back(EncodeSMovB32(9, InlineU32(3))); + code.push_back(EncodeSop2(0x06, 10, 8, 9)); + code.push_back(EncodeSop2(0x08, 11, 8, 9)); + AppendSMovLiteral(&code, 12, 0xfffffffeu); + code.push_back(EncodeSMovB32(13, InlineU32(3))); + code.push_back(EncodeSop2(0x07, 14, 12, 13)); + code.push_back(EncodeSop2(0x09, 15, 12, 13)); + code.push_back(EncodeSMovB32(16, 8)); + code.push_back(EncodeSop1(0x34, 16, 16)); + code.push_back(EncodeSopk(0x00, 17, 0xfff5u)); + code.push_back(EncodeSopk(0x10, 17, 0xfffdu)); - const u32 results[] = {2, 5, 6, 7, 10, 11, 14, 15, 16, 17}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreSgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {2, 5, 6, 7, 10, 11, 14, 15, 16, 17}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreSgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"ScalarExtendedArithmetic", - code, - {}, - {0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33}, - {O::SMovB32, O::SAddU32, O::SAddcU32, O::SAddI32, O::SSubI32, O::SMinI32, O::SMaxI32, - O::SMinU32, O::SMaxU32, O::SAbsI32, O::SMovkI32, O::SMulkI32, O::VMovB32, - O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarExtendedArithmetic", + code, + {}, + {0, 12, 15, 0xfffffffeu, 0xfffffffbu, 3, 3, 0xfffffffeu, 5, 33}, + {O::SMovB32, O::SAddU32, O::SAddcU32, O::SAddI32, O::SSubI32, + O::SMinI32, O::SMaxI32, O::SMinU32, O::SMaxU32, O::SAbsI32, + O::SMovkI32, O::SMulkI32, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase ScalarArithmeticSccCarryBorrowOverflow() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - auto set_scc = [&](bool value) { - code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); - }; - auto capture_scc = [&](u32 dst_sgpr) { - code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); - }; - auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { - set_scc(prior_scc); - code.push_back(instruction); - capture_scc(out_sgpr); - }; + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { + set_scc(prior_scc); + code.push_back(instruction); + capture_scc(out_sgpr); + }; - code.push_back(EncodeSMovB32(0, InlineU32(0))); - code.push_back(EncodeSMovB32(1, InlineU32(1))); - append_case(false, EncodeSop2(0x01, 10, 0, 1), 20); + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + append_case(false, EncodeSop2(0x01, 10, 0, 1), 20); - code.push_back(EncodeSMovB32(2, InlineU32(5))); - code.push_back(EncodeSMovB32(3, InlineU32(3))); - append_case(true, EncodeSop2(0x01, 11, 2, 3), 21); + code.push_back(EncodeSMovB32(2, InlineU32(5))); + code.push_back(EncodeSMovB32(3, InlineU32(3))); + append_case(true, EncodeSop2(0x01, 11, 2, 3), 21); - AppendSMovLiteral(&code, 4, 0x7fffffffu); - code.push_back(EncodeSMovB32(5, InlineU32(1))); - append_case(false, EncodeSop2(0x02, 12, 4, 5), 22); + AppendSMovLiteral(&code, 4, 0x7fffffffu); + code.push_back(EncodeSMovB32(5, InlineU32(1))); + append_case(false, EncodeSop2(0x02, 12, 4, 5), 22); - code.push_back(EncodeSMovB32(6, InlineU32(1))); - code.push_back(EncodeSMovB32(7, InlineU32(2))); - append_case(true, EncodeSop2(0x02, 13, 6, 7), 23); + code.push_back(EncodeSMovB32(6, InlineU32(1))); + code.push_back(EncodeSMovB32(7, InlineU32(2))); + append_case(true, EncodeSop2(0x02, 13, 6, 7), 23); - AppendSMovLiteral(&code, 8, 0x80000000u); - code.push_back(EncodeSMovB32(9, InlineU32(1))); - append_case(false, EncodeSop2(0x03, 14, 8, 9), 24); + AppendSMovLiteral(&code, 8, 0x80000000u); + code.push_back(EncodeSMovB32(9, InlineU32(1))); + append_case(false, EncodeSop2(0x03, 14, 8, 9), 24); - code.push_back(EncodeSMovB32(15, InlineU32(5))); - code.push_back(EncodeSMovB32(16, InlineU32(3))); - append_case(true, EncodeSop2(0x03, 17, 15, 16), 25); + code.push_back(EncodeSMovB32(15, InlineU32(5))); + code.push_back(EncodeSMovB32(16, InlineU32(3))); + append_case(true, EncodeSop2(0x03, 17, 15, 16), 25); - AppendSMovLiteral(&code, 18, 0x7fffffffu); - set_scc(false); - code.push_back(EncodeSopk(0x0f, 18, 1)); - capture_scc(26); + AppendSMovLiteral(&code, 18, 0x7fffffffu); + set_scc(false); + code.push_back(EncodeSopk(0x0f, 18, 1)); + capture_scc(26); - code.push_back(EncodeSMovB32(19, InlineU32(4))); - set_scc(true); - code.push_back(EncodeSopk(0x0f, 19, 0xfffeu)); - capture_scc(27); + code.push_back(EncodeSMovB32(19, InlineU32(4))); + set_scc(true); + code.push_back(EncodeSopk(0x0f, 19, 0xfffeu)); + capture_scc(27); - for (u32 i = 0; i < 8u; i++) { - AppendStoreSgpr(&code, 20u + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 8u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); - return {"ScalarArithmeticSccCarryBorrowOverflow", - code, - {}, - {1, 0, 1, 0, 1, 0, 1, 0}, - {O::SMovB32, O::SSubU32, O::SAddI32, O::SSubI32, O::SCmpEqU32, O::SCselectB32, - O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarArithmeticSccCarryBorrowOverflow", + code, + {}, + {1, 0, 1, 0, 1, 0, 1, 0}, + {O::SMovB32, O::SSubU32, O::SAddI32, O::SSubI32, O::SCmpEqU32, + O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarMinMaxSccComparisonEdges() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - auto set_scc = [&](bool value) { - code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); - }; - auto capture_scc = [&](u32 dst_sgpr) { - code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); - }; - auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { - set_scc(prior_scc); - code.push_back(instruction); - capture_scc(out_sgpr); - }; + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { + set_scc(prior_scc); + code.push_back(instruction); + capture_scc(out_sgpr); + }; - AppendSMovLiteral(&code, 0, 0xfffffffeu); - code.push_back(EncodeSMovB32(1, InlineU32(3))); - append_case(false, EncodeSop2(0x06, 10, 0, 1), 20); - append_case(true, EncodeSop2(0x06, 11, 1, 0), 21); - append_case(false, EncodeSop2(0x08, 12, 1, 0), 22); - append_case(true, EncodeSop2(0x08, 13, 0, 1), 23); + AppendSMovLiteral(&code, 0, 0xfffffffeu); + code.push_back(EncodeSMovB32(1, InlineU32(3))); + append_case(false, EncodeSop2(0x06, 10, 0, 1), 20); + append_case(true, EncodeSop2(0x06, 11, 1, 0), 21); + append_case(false, EncodeSop2(0x08, 12, 1, 0), 22); + append_case(true, EncodeSop2(0x08, 13, 0, 1), 23); - code.push_back(EncodeSMovB32(2, InlineU32(2))); - code.push_back(EncodeSMovB32(3, InlineU32(3))); - append_case(false, EncodeSop2(0x07, 14, 2, 3), 24); - append_case(true, EncodeSop2(0x07, 15, 3, 2), 25); - append_case(false, EncodeSop2(0x09, 16, 3, 2), 26); - append_case(true, EncodeSop2(0x09, 17, 2, 3), 27); + code.push_back(EncodeSMovB32(2, InlineU32(2))); + code.push_back(EncodeSMovB32(3, InlineU32(3))); + append_case(false, EncodeSop2(0x07, 14, 2, 3), 24); + append_case(true, EncodeSop2(0x07, 15, 3, 2), 25); + append_case(false, EncodeSop2(0x09, 16, 3, 2), 26); + append_case(true, EncodeSop2(0x09, 17, 2, 3), 27); - for (u32 i = 0; i < 8u; i++) { - AppendStoreSgpr(&code, 20u + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 8u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); - return {"ScalarMinMaxSccComparisonEdges", - code, - {}, - {1, 0, 1, 0, 1, 0, 1, 0}, - {O::SMovB32, O::SMinI32, O::SMaxI32, O::SMinU32, O::SMaxU32, O::SCmpEqU32, - O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarMinMaxSccComparisonEdges", + code, + {}, + {1, 0, 1, 0, 1, 0, 1, 0}, + {O::SMovB32, O::SMinI32, O::SMaxI32, O::SMinU32, O::SMaxU32, + O::SCmpEqU32, O::SCselectB32, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase ScalarAbsI32UpdatesScc() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - auto set_scc = [&](bool value) { - code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); - }; - auto capture_scc = [&](u32 dst_sgpr) { - code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); - }; + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; - code.push_back(EncodeSMovB32(0, InlineU32(0))); - set_scc(true); - code.push_back(EncodeSop1(0x34, 1, 0)); - capture_scc(2); + code.push_back(EncodeSMovB32(0, InlineU32(0))); + set_scc(true); + code.push_back(EncodeSop1(0x34, 1, 0)); + capture_scc(2); - AppendSMovLiteral(&code, 3, 0xfffffffbu); - set_scc(false); - code.push_back(EncodeSop1(0x34, 4, 3)); - capture_scc(5); + AppendSMovLiteral(&code, 3, 0xfffffffbu); + set_scc(false); + code.push_back(EncodeSop1(0x34, 4, 3)); + capture_scc(5); - AppendStoreSgpr(&code, 1, 0); - AppendStoreSgpr(&code, 2, 1); - AppendStoreSgpr(&code, 4, 2); - AppendStoreSgpr(&code, 5, 3); - AppendEnd(&code); + AppendStoreSgpr(&code, 1, 0); + AppendStoreSgpr(&code, 2, 1); + AppendStoreSgpr(&code, 4, 2); + AppendStoreSgpr(&code, 5, 3); + AppendEnd(&code); - return {"ScalarAbsI32UpdatesScc", - code, - {}, - {0, 0, 5, 1}, - {O::SMovB32, O::SCmpEqU32, O::SAbsI32, O::SCselectB32, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarAbsI32UpdatesScc", + code, + {}, + {0, 0, 5, 1}, + {O::SMovB32, O::SCmpEqU32, O::SAbsI32, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarShiftLeftAddSccCarryEdges() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - auto set_scc = [&](bool value) { - code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); - }; - auto capture_scc = [&](u32 dst_sgpr) { - code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); - }; - auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { - set_scc(prior_scc); - code.push_back(instruction); - capture_scc(out_sgpr); - }; + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; + auto append_case = [&](bool prior_scc, u32 instruction, u32 out_sgpr) { + set_scc(prior_scc); + code.push_back(instruction); + capture_scc(out_sgpr); + }; - AppendSMovLiteral(&code, 0, 0x80000000u); - AppendSMovLiteral(&code, 1, 0x40000000u); - AppendSMovLiteral(&code, 2, 0x20000000u); - AppendSMovLiteral(&code, 3, 0x10000000u); - code.push_back(EncodeSMovB32(4, InlineU32(1))); - code.push_back(EncodeSMovB32(5, InlineU32(0))); - append_case(false, EncodeSop2(0x2e, 10, 0, 5), 20); - append_case(true, EncodeSop2(0x2e, 11, 4, 4), 21); - append_case(false, EncodeSop2(0x2f, 12, 1, 5), 22); - append_case(true, EncodeSop2(0x2f, 13, 4, 4), 23); - append_case(false, EncodeSop2(0x30, 14, 2, 5), 24); - append_case(true, EncodeSop2(0x30, 15, 4, 4), 25); - append_case(false, EncodeSop2(0x31, 16, 3, 5), 26); - append_case(true, EncodeSop2(0x31, 17, 4, 4), 27); + AppendSMovLiteral(&code, 0, 0x80000000u); + AppendSMovLiteral(&code, 1, 0x40000000u); + AppendSMovLiteral(&code, 2, 0x20000000u); + AppendSMovLiteral(&code, 3, 0x10000000u); + code.push_back(EncodeSMovB32(4, InlineU32(1))); + code.push_back(EncodeSMovB32(5, InlineU32(0))); + append_case(false, EncodeSop2(0x2e, 10, 0, 5), 20); + append_case(true, EncodeSop2(0x2e, 11, 4, 4), 21); + append_case(false, EncodeSop2(0x2f, 12, 1, 5), 22); + append_case(true, EncodeSop2(0x2f, 13, 4, 4), 23); + append_case(false, EncodeSop2(0x30, 14, 2, 5), 24); + append_case(true, EncodeSop2(0x30, 15, 4, 4), 25); + append_case(false, EncodeSop2(0x31, 16, 3, 5), 26); + append_case(true, EncodeSop2(0x31, 17, 4, 4), 27); - for (u32 i = 0; i < 8u; i++) { - AppendStoreSgpr(&code, 20u + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 8u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); - return {"ScalarShiftLeftAddSccCarryEdges", - code, - {}, - {1, 0, 1, 0, 1, 0, 1, 0}, - {O::SMovB32, O::SLshl1AddU32, O::SLshl2AddU32, O::SLshl3AddU32, O::SLshl4AddU32, - O::SCmpEqU32, O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarShiftLeftAddSccCarryEdges", + code, + {}, + {1, 0, 1, 0, 1, 0, 1, 0}, + {O::SMovB32, O::SLshl1AddU32, O::SLshl2AddU32, O::SLshl3AddU32, + O::SLshl4AddU32, O::SCmpEqU32, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarCompareOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 0xfffffffeu); - code.push_back(EncodeSMovB32(1, InlineU32(1))); - code.push_back(EncodeSMovB32(2, InlineU32(5))); - code.push_back(EncodeSMovB32(3, InlineU32(2))); - code.push_back(EncodeSMovB32(4, InlineU32(3))); - AppendSMovLiteral(&code, 10, 0x00000001u); - code.push_back(EncodeSMovB32(11, InlineU32(0))); - code.push_back(EncodeSMovB32(12, InlineU32(2))); - code.push_back(EncodeSMovB32(13, InlineU32(0))); - code.push_back(EncodeSMovB32(14, InlineU32(1))); - code.push_back(EncodeSMovB32(15, InlineU32(1))); + std::vector code; + AppendSMovLiteral(&code, 0, 0xfffffffeu); + code.push_back(EncodeSMovB32(1, InlineU32(1))); + code.push_back(EncodeSMovB32(2, InlineU32(5))); + code.push_back(EncodeSMovB32(3, InlineU32(2))); + code.push_back(EncodeSMovB32(4, InlineU32(3))); + AppendSMovLiteral(&code, 10, 0x00000001u); + code.push_back(EncodeSMovB32(11, InlineU32(0))); + code.push_back(EncodeSMovB32(12, InlineU32(2))); + code.push_back(EncodeSMovB32(13, InlineU32(0))); + code.push_back(EncodeSMovB32(14, InlineU32(1))); + code.push_back(EncodeSMovB32(15, InlineU32(1))); - u32 dst = 20; - auto append_compare = [&](u32 opcode, u32 src0, u32 src1) { - code.push_back(EncodeSopc(opcode, src0, src1)); - code.push_back(EncodeSop2(0x0a, dst++, InlineU32(1), InlineU32(0))); - }; + u32 dst = 20; + auto append_compare = [&](u32 opcode, u32 src0, u32 src1) { + code.push_back(EncodeSopc(opcode, src0, src1)); + code.push_back(EncodeSop2(0x0a, dst++, InlineU32(1), InlineU32(0))); + }; - append_compare(0x00, 2, 2); - append_compare(0x01, 0, 1); - append_compare(0x02, 1, 0); - append_compare(0x03, 1, 1); - append_compare(0x04, 0, 1); - append_compare(0x05, 0, 0); - append_compare(0x07, 3, 4); - append_compare(0x08, 4, 3); - append_compare(0x09, 4, 4); - append_compare(0x0b, 3, 4); - append_compare(0x12, 10, 10); - append_compare(0x12, 10, 12); - append_compare(0x12, 10, 14); - append_compare(0x13, 10, 12); - append_compare(0x13, 10, 14); + append_compare(0x00, 2, 2); + append_compare(0x01, 0, 1); + append_compare(0x02, 1, 0); + append_compare(0x03, 1, 1); + append_compare(0x04, 0, 1); + append_compare(0x05, 0, 0); + append_compare(0x07, 3, 4); + append_compare(0x08, 4, 3); + append_compare(0x09, 4, 4); + append_compare(0x0b, 3, 4); + append_compare(0x12, 10, 10); + append_compare(0x12, 10, 12); + append_compare(0x12, 10, 14); + append_compare(0x13, 10, 12); + append_compare(0x13, 10, 14); - for (u32 i = 0; i < 15u; i++) { - AppendStoreSgpr(&code, 20u + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 15u; i++) { + AppendStoreSgpr(&code, 20u + i, i); + } + AppendEnd(&code); - return {"ScalarCompareOps", - code, - {}, - {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1}, - {O::SMovB32, O::SCmpEqI32, O::SCmpLgI32, O::SCmpGtI32, O::SCmpGeI32, O::SCmpLtI32, - O::SCmpLeI32, O::SCmpLgU32, O::SCmpGtU32, O::SCmpGeU32, O::SCmpLeU32, O::SCmpEqU64, - O::SCmpLgU64, O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarCompareOps", + code, + {}, + {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1}, + {O::SMovB32, O::SCmpEqI32, O::SCmpLgI32, O::SCmpGtI32, O::SCmpGeI32, + O::SCmpLtI32, O::SCmpLeI32, O::SCmpLgU32, O::SCmpGtU32, O::SCmpGeU32, + O::SCmpLeU32, O::SCmpEqU64, O::SCmpLgU64, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarShiftAddAndMaskOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(2))); - code.push_back(EncodeSMovB32(1, InlineU32(3))); - code.push_back(EncodeSop2(0x2e, 2, 0, 1)); - code.push_back(EncodeSop2(0x2f, 3, 0, 1)); - code.push_back(EncodeSop2(0x30, 4, 0, 1)); - code.push_back(EncodeSop2(0x31, 5, 0, 1)); - AppendSMovLiteral(&code, 6, 0xfffffff8u); - code.push_back(EncodeSop2(0x22, 7, 6, InlineU32(2))); - AppendSMovLiteral(&code, 8, 0xffffffffu); - code.push_back(EncodeSMovB32(9, InlineU32(2))); - code.push_back(EncodeSop2(0x35, 10, 8, 9)); - AppendSMovLiteral(&code, 12, 0x0f0f0f0fu); - AppendSMovLiteral(&code, 13, 0x00ff00ffu); - code.push_back(EncodeSop1(0x08, 14, 12)); - code.push_back(EncodeSop1(0x0a, 16, 12)); + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(2))); + code.push_back(EncodeSMovB32(1, InlineU32(3))); + code.push_back(EncodeSop2(0x2e, 2, 0, 1)); + code.push_back(EncodeSop2(0x2f, 3, 0, 1)); + code.push_back(EncodeSop2(0x30, 4, 0, 1)); + code.push_back(EncodeSop2(0x31, 5, 0, 1)); + AppendSMovLiteral(&code, 6, 0xfffffff8u); + code.push_back(EncodeSop2(0x22, 7, 6, InlineU32(2))); + AppendSMovLiteral(&code, 8, 0xffffffffu); + code.push_back(EncodeSMovB32(9, InlineU32(2))); + code.push_back(EncodeSop2(0x35, 10, 8, 9)); + AppendSMovLiteral(&code, 12, 0x0f0f0f0fu); + AppendSMovLiteral(&code, 13, 0x00ff00ffu); + code.push_back(EncodeSop1(0x08, 14, 12)); + code.push_back(EncodeSop1(0x0a, 16, 12)); - AppendStoreSgpr(&code, 2, 0); - AppendStoreSgpr(&code, 3, 1); - AppendStoreSgpr(&code, 4, 2); - AppendStoreSgpr(&code, 5, 3); - AppendStoreSgpr(&code, 7, 4); - AppendStoreSgpr(&code, 10, 5); - AppendStoreSgprPair(&code, 14, 6); - AppendStoreSgprPair(&code, 16, 8); - AppendEnd(&code); + AppendStoreSgpr(&code, 2, 0); + AppendStoreSgpr(&code, 3, 1); + AppendStoreSgpr(&code, 4, 2); + AppendStoreSgpr(&code, 5, 3); + AppendStoreSgpr(&code, 7, 4); + AppendStoreSgpr(&code, 10, 5); + AppendStoreSgprPair(&code, 14, 6); + AppendStoreSgprPair(&code, 16, 8); + AppendEnd(&code); - return {"ScalarShiftAddAndMaskOps", - code, - {}, - {7, 11, 19, 35, 0xfffffffeu, 1, 0xf0f0f0f0u, 0xff00ff00u, 0x0f0f0f0fu, 0x00ff00ffu}, - {O::SMovB32, O::SLshl1AddU32, O::SLshl2AddU32, O::SLshl3AddU32, O::SLshl4AddU32, - O::SAshrI32, O::SMulHiU32, O::SNotB64, O::SWqmB64, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarShiftAddAndMaskOps", + code, + {}, + {7, 11, 19, 35, 0xfffffffeu, 1, 0xf0f0f0f0u, 0xff00ff00u, 0x0f0f0f0fu, + 0x00ff00ffu}, + {O::SMovB32, O::SLshl1AddU32, O::SLshl2AddU32, O::SLshl3AddU32, + O::SLshl4AddU32, O::SAshrI32, O::SMulHiU32, O::SNotB64, O::SWqmB64, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarNotB64UpdatesScc() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - auto set_scc = [&](bool value) { - code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); - }; - auto capture_scc = [&](u32 dst_sgpr) { - code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); - }; + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; - AppendSMovLiteral(&code, 0, 0xffffffffu); - AppendSMovLiteral(&code, 1, 0xffffffffu); - set_scc(true); - code.push_back(EncodeSop1(0x08, 2, 0)); - capture_scc(4); + AppendSMovLiteral(&code, 0, 0xffffffffu); + AppendSMovLiteral(&code, 1, 0xffffffffu); + set_scc(true); + code.push_back(EncodeSop1(0x08, 2, 0)); + capture_scc(4); - AppendSMovLiteral(&code, 6, 0xfffffffeu); - AppendSMovLiteral(&code, 7, 0xffffffffu); - set_scc(false); - code.push_back(EncodeSop1(0x08, 8, 6)); - capture_scc(10); + AppendSMovLiteral(&code, 6, 0xfffffffeu); + AppendSMovLiteral(&code, 7, 0xffffffffu); + set_scc(false); + code.push_back(EncodeSop1(0x08, 8, 6)); + capture_scc(10); - AppendStoreSgpr(&code, 4, 0); - AppendStoreSgpr(&code, 10, 1); - AppendStoreSgprPair(&code, 2, 2); - AppendStoreSgprPair(&code, 8, 4); - AppendEnd(&code); + AppendStoreSgpr(&code, 4, 0); + AppendStoreSgpr(&code, 10, 1); + AppendStoreSgprPair(&code, 2, 2); + AppendStoreSgprPair(&code, 8, 4); + AppendEnd(&code); - return {"ScalarNotB64UpdatesScc", - code, - {}, - {0, 1, 0, 0, 1, 0}, - {O::SMovB32, O::SCmpEqU32, O::SNotB64, O::SCselectB32, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarNotB64UpdatesScc", + code, + {}, + {0, 1, 0, 0, 1, 0}, + {O::SMovB32, O::SCmpEqU32, O::SNotB64, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarFlbitI32B64Gpu() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 0x00000000u); - AppendSMovLiteral(&code, 1, 0x80000000u); - code.push_back(EncodeSop1(0x16, 20, 0)); + std::vector code; + AppendSMovLiteral(&code, 0, 0x00000000u); + AppendSMovLiteral(&code, 1, 0x80000000u); + code.push_back(EncodeSop1(0x16, 20, 0)); - AppendSMovLiteral(&code, 2, 0x00008000u); - AppendSMovLiteral(&code, 3, 0x00000000u); - code.push_back(EncodeSop1(0x16, 21, 2)); + AppendSMovLiteral(&code, 2, 0x00008000u); + AppendSMovLiteral(&code, 3, 0x00000000u); + code.push_back(EncodeSop1(0x16, 21, 2)); - AppendSMovLiteral(&code, 4, 0x00000000u); - AppendSMovLiteral(&code, 5, 0x00000000u); - code.push_back(EncodeSop1(0x16, 22, 4)); + AppendSMovLiteral(&code, 4, 0x00000000u); + AppendSMovLiteral(&code, 5, 0x00000000u); + code.push_back(EncodeSop1(0x16, 22, 4)); - AppendSMovLiteral(&code, 14, 0x00000008u); - AppendSMovLiteral(&code, 15, 0x00000000u); - code.push_back(EncodeSop1(0x16, 106, 14)); + AppendSMovLiteral(&code, 14, 0x00000008u); + AppendSMovLiteral(&code, 15, 0x00000000u); + code.push_back(EncodeSop1(0x16, 106, 14)); - AppendStoreSgpr(&code, 20, 0); - AppendStoreSgpr(&code, 21, 1); - AppendStoreSgpr(&code, 22, 2); - AppendStoreSgpr(&code, 106, 3); - AppendEnd(&code); + AppendStoreSgpr(&code, 20, 0); + AppendStoreSgpr(&code, 21, 1); + AppendStoreSgpr(&code, 22, 2); + AppendStoreSgpr(&code, 106, 3); + AppendEnd(&code); - return {"ScalarFlbitI32B64Gpu", - code, - {}, - {0, 48, 0xffffffffu, 60}, - {O::SMovB32, O::SFlbitI32B64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarFlbitI32B64Gpu", + code, + {}, + {0, 48, 0xffffffffu, 60}, + {O::SMovB32, O::SFlbitI32B64, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase ScalarSaveExecOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 106, 0xffffffffu); - AppendSMovLiteral(&code, 107, 0xffffffffu); - code.push_back(0xbe80246au); - code.push_back(EncodeSop1(0x28, 2, 106)); - code.push_back(EncodeSMovB32(106, InlineU32(0))); - code.push_back(EncodeSMovB32(107, InlineU32(0))); - code.push_back(EncodeSop1(0x37, 4, 106)); - AppendSMovLiteral(&code, 106, 0x00000003u); - code.push_back(EncodeSop1(0x3c, 8, 106)); - code.push_back(EncodeSop1(0x04, 10, 126)); - code.push_back(EncodeSMovB32(106, InlineU32(1))); - code.push_back(EncodeSop1(0x44, 12, 106)); - code.push_back(EncodeSop1(0x04, 14, 126)); - AppendSMovLiteral(&code, 126, 0xffffffffu); - AppendSMovLiteral(&code, 127, 0xffffffffu); + std::vector code; + AppendSMovLiteral(&code, 106, 0xffffffffu); + AppendSMovLiteral(&code, 107, 0xffffffffu); + code.push_back(0xbe80246au); + code.push_back(EncodeSop1(0x28, 2, 106)); + code.push_back(EncodeSMovB32(106, InlineU32(0))); + code.push_back(EncodeSMovB32(107, InlineU32(0))); + code.push_back(EncodeSop1(0x37, 4, 106)); + AppendSMovLiteral(&code, 106, 0x00000003u); + code.push_back(EncodeSop1(0x3c, 8, 106)); + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeSMovB32(106, InlineU32(1))); + code.push_back(EncodeSop1(0x44, 12, 106)); + code.push_back(EncodeSop1(0x04, 14, 126)); + AppendSMovLiteral(&code, 126, 0xffffffffu); + AppendSMovLiteral(&code, 127, 0xffffffffu); - AppendStoreSgprPair(&code, 0, 0); - AppendStoreSgprPair(&code, 2, 2); - AppendStoreSgprPair(&code, 4, 4); - AppendStoreSgpr(&code, 8, 6); - AppendStoreSgprPair(&code, 10, 7); - AppendStoreSgpr(&code, 12, 9); - AppendStoreSgprPair(&code, 14, 10); - AppendStoreSgpr(&code, 253, 12); - AppendEnd(&code); + AppendStoreSgprPair(&code, 0, 0); + AppendStoreSgprPair(&code, 2, 2); + AppendStoreSgprPair(&code, 4, 4); + AppendStoreSgpr(&code, 8, 6); + AppendStoreSgprPair(&code, 10, 7); + AppendStoreSgpr(&code, 12, 9); + AppendStoreSgprPair(&code, 14, 10); + AppendStoreSgpr(&code, 253, 12); + AppendEnd(&code); - return {"ScalarSaveExecOps", - code, - {}, - {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, - 0xffffffffu, 0x00000003u, 0xffffffffu, 0x00000003u, 0x00000002u, 0xffffffffu, 1}, - {O::SMovB32, O::SAndSaveexecB64, O::SOrn2SaveexecB64, O::SAndn1SaveexecB64, - O::SAndSaveexecB32, O::SAndn1SaveexecB32, O::SMovB64, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarSaveExecOps", + code, + {}, + {0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, + 0xffffffffu, 0xffffffffu, 0x00000003u, 0xffffffffu, 0x00000003u, + 0x00000002u, 0xffffffffu, 1}, + {O::SMovB32, O::SAndSaveexecB64, O::SOrn2SaveexecB64, + O::SAndn1SaveexecB64, O::SAndSaveexecB32, O::SAndn1SaveexecB32, + O::SMovB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarOrn2SaveexecUsesSourceOrNotExec() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 126, 0x0000000cu); - AppendSMovLiteral(&code, 127, 0x80000000u); - AppendSMovLiteral(&code, 0, 0x00000001u); - AppendSMovLiteral(&code, 1, 0x00000001u); - code.push_back(EncodeSop1(0x28, 2, 0)); - code.push_back(EncodeSop1(0x04, 4, 126)); - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); - AppendStoreSgprPair(&code, 2, 0); - AppendStoreSgprPair(&code, 4, 2); - AppendStoreSgpr(&code, 253, 4); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 126, 0x0000000cu); + AppendSMovLiteral(&code, 127, 0x80000000u); + AppendSMovLiteral(&code, 0, 0x00000001u); + AppendSMovLiteral(&code, 1, 0x00000001u); + code.push_back(EncodeSop1(0x28, 2, 0)); + code.push_back(EncodeSop1(0x04, 4, 126)); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + AppendStoreSgprPair(&code, 2, 0); + AppendStoreSgprPair(&code, 4, 2); + AppendStoreSgpr(&code, 253, 4); + AppendEnd(&code); - return { - "ScalarOrn2SaveexecUsesSourceOrNotExec", - code, - {}, - {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1}, - {O::SMovB32, O::SOrn2SaveexecB64, O::SMovB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarOrn2SaveexecUsesSourceOrNotExec", + code, + {}, + {0x0000000cu, 0x80000000u, 0xfffffff3u, 0x7fffffffu, 1}, + {O::SMovB32, O::SOrn2SaveexecB64, O::SMovB64, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarGetpcWritesNextInstructionPc() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSop1(0x1f, 0, 0)); - AppendStoreSgprPair(&code, 0, 0); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSop1(0x1f, 0, 0)); + AppendStoreSgprPair(&code, 0, 0); + AppendEnd(&code); - return {"ScalarGetpcWritesNextInstructionPc", - code, - {}, - {4, 0}, - {O::SGetpcB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarGetpcWritesNextInstructionPc", + code, + {}, + {4, 0}, + {O::SGetpcB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarBitfieldPack() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(15))); - code.push_back(EncodeSop1(0x0b, 1, 0)); - AppendSMovLiteral(&code, 2, 0xf0000000u); - code.push_back(EncodeSop1(0x10, 3, 0)); - AppendSMovLiteral(&code, 19, 0xf0f00001u); - code.push_back(EncodeSop1(0x0f, 20, 19)); - code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0))); - code.push_back(EncodeSMovB32(22, InlineU32(0))); - code.push_back(EncodeSop1(0x0f, 23, 22)); - code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0))); - code.push_back(EncodeSMovB32(4, InlineU32(3))); - code.push_back(EncodeSop1(0x3b, 5, 4)); - code.push_back(EncodeSop2(0x24, 7, InlineU32(4), InlineU32(8))); - AppendSMovLiteral(&code, 8, 0x00f00000u); - AppendSMovLiteral(&code, 9, 0x00040014u); - code.push_back(EncodeSop2(0x27, 10, 8, 9)); - AppendSMovLiteral(&code, 11, 0xaaaabbbbu); - AppendSMovLiteral(&code, 12, 0xccccddddu); - code.push_back(EncodeSop2(0x32, 13, 11, 12)); - code.push_back(EncodeSop2(0x33, 14, 11, 12)); - code.push_back(EncodeSop2(0x34, 15, 11, 12)); - code.push_back(EncodeSopc(0x0d, InlineU32(4), InlineU32(2))); - code.push_back(EncodeSop2(0x0a, 17, InlineU32(1), InlineU32(0))); - code.push_back(EncodeSopc(0x0c, InlineU32(4), InlineU32(1))); - code.push_back(EncodeSop2(0x0a, 18, InlineU32(1), InlineU32(0))); + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(15))); + code.push_back(EncodeSop1(0x0b, 1, 0)); + AppendSMovLiteral(&code, 2, 0xf0000000u); + code.push_back(EncodeSop1(0x10, 3, 0)); + AppendSMovLiteral(&code, 19, 0xf0f00001u); + code.push_back(EncodeSop1(0x0f, 20, 19)); + code.push_back(EncodeSop2(0x0a, 21, InlineU32(1), InlineU32(0))); + code.push_back(EncodeSMovB32(22, InlineU32(0))); + code.push_back(EncodeSop1(0x0f, 23, 22)); + code.push_back(EncodeSop2(0x0a, 24, InlineU32(1), InlineU32(0))); + code.push_back(EncodeSMovB32(4, InlineU32(3))); + code.push_back(EncodeSop1(0x3b, 5, 4)); + code.push_back(EncodeSop2(0x24, 7, InlineU32(4), InlineU32(8))); + AppendSMovLiteral(&code, 8, 0x00f00000u); + AppendSMovLiteral(&code, 9, 0x00040014u); + code.push_back(EncodeSop2(0x27, 10, 8, 9)); + AppendSMovLiteral(&code, 11, 0xaaaabbbbu); + AppendSMovLiteral(&code, 12, 0xccccddddu); + code.push_back(EncodeSop2(0x32, 13, 11, 12)); + code.push_back(EncodeSop2(0x33, 14, 11, 12)); + code.push_back(EncodeSop2(0x34, 15, 11, 12)); + code.push_back(EncodeSopc(0x0d, InlineU32(4), InlineU32(2))); + code.push_back(EncodeSop2(0x0a, 17, InlineU32(1), InlineU32(0))); + code.push_back(EncodeSopc(0x0c, InlineU32(4), InlineU32(1))); + code.push_back(EncodeSop2(0x0a, 18, InlineU32(1), InlineU32(0))); - AppendStoreSgpr(&code, 1, 0); - AppendStoreSgpr(&code, 3, 1); - AppendStoreSgpr(&code, 20, 2); - AppendStoreSgpr(&code, 21, 3); - AppendStoreSgpr(&code, 23, 4); - AppendStoreSgpr(&code, 24, 5); - AppendStoreSgprPair(&code, 5, 6); - AppendStoreSgpr(&code, 7, 8); - AppendStoreSgpr(&code, 10, 9); - AppendStoreSgpr(&code, 13, 10); - AppendStoreSgpr(&code, 14, 11); - AppendStoreSgpr(&code, 15, 12); - AppendStoreSgpr(&code, 17, 13); - AppendStoreSgpr(&code, 18, 14); - AppendEnd(&code); + AppendStoreSgpr(&code, 1, 0); + AppendStoreSgpr(&code, 3, 1); + AppendStoreSgpr(&code, 20, 2); + AppendStoreSgpr(&code, 21, 3); + AppendStoreSgpr(&code, 23, 4); + AppendStoreSgpr(&code, 24, 5); + AppendStoreSgprPair(&code, 5, 6); + AppendStoreSgpr(&code, 7, 8); + AppendStoreSgpr(&code, 10, 9); + AppendStoreSgpr(&code, 13, 10); + AppendStoreSgpr(&code, 14, 11); + AppendStoreSgpr(&code, 15, 12); + AppendStoreSgpr(&code, 17, 13); + AppendStoreSgpr(&code, 18, 14); + AppendEnd(&code); - return {"ScalarBitfieldPack", - code, - {}, - {0xf0000000u, 8, 9, 1, 0, 0, 0x0000000fu, 0, 0x00000f00u, 0x0000000fu, 0xddddbbbbu, - 0xccccbbbbu, 0xccccaaaau, 1, 1}, - {O::SMovB32, O::SBrevB32, O::SBcnt1I32B32, O::SBcnt1I32B64, O::SBitreplicateB64B32, - O::SBfmB32, O::SBfeU32, O::SPackLlB32B16, O::SPackLhB32B16, O::SPackHhB32B16, - O::SBitcmp0B32, O::SBitcmp1B32, O::SCselectB32, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarBitfieldPack", + code, + {}, + {0xf0000000u, 8, 9, 1, 0, 0, 0x0000000fu, 0, 0x00000f00u, 0x0000000fu, + 0xddddbbbbu, 0xccccbbbbu, 0xccccaaaau, 1, 1}, + {O::SMovB32, O::SBrevB32, O::SBcnt1I32B32, O::SBcnt1I32B64, + O::SBitreplicateB64B32, O::SBfmB32, O::SBfeU32, O::SPackLlB32B16, + O::SPackLhB32B16, O::SPackHhB32B16, O::SBitcmp0B32, O::SBitcmp1B32, + O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarBrevB32PreservesScc() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - auto set_scc = [&](bool value) { - code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); - }; - auto capture_scc = [&](u32 dst_sgpr) { - code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); - }; + std::vector code; + auto set_scc = [&](bool value) { + code.push_back(EncodeSopc(0x06, InlineU32(1), InlineU32(value ? 1u : 0u))); + }; + auto capture_scc = [&](u32 dst_sgpr) { + code.push_back(EncodeSop2(0x0a, dst_sgpr, InlineU32(1), InlineU32(0))); + }; - code.push_back(EncodeSMovB32(0, InlineU32(1))); - set_scc(false); - code.push_back(EncodeSop1(0x0b, 1, 0)); - capture_scc(2); + code.push_back(EncodeSMovB32(0, InlineU32(1))); + set_scc(false); + code.push_back(EncodeSop1(0x0b, 1, 0)); + capture_scc(2); - code.push_back(EncodeSMovB32(3, InlineU32(0))); - set_scc(true); - code.push_back(EncodeSop1(0x0b, 4, 3)); - capture_scc(5); + code.push_back(EncodeSMovB32(3, InlineU32(0))); + set_scc(true); + code.push_back(EncodeSop1(0x0b, 4, 3)); + capture_scc(5); - AppendStoreSgpr(&code, 1, 0); - AppendStoreSgpr(&code, 2, 1); - AppendStoreSgpr(&code, 4, 2); - AppendStoreSgpr(&code, 5, 3); - AppendEnd(&code); + AppendStoreSgpr(&code, 1, 0); + AppendStoreSgpr(&code, 2, 1); + AppendStoreSgpr(&code, 4, 2); + AppendStoreSgpr(&code, 5, 3); + AppendEnd(&code); - return {"ScalarBrevB32PreservesScc", - code, - {}, - {0x80000000u, 0, 0, 1}, - {O::SMovB32, O::SCmpEqU32, O::SBrevB32, O::SCselectB32, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarBrevB32PreservesScc", + code, + {}, + {0x80000000u, 0, 0, 1}, + {O::SMovB32, O::SCmpEqU32, O::SBrevB32, O::SCselectB32, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase BitfieldExtractWidthPastEndEdges() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 0xf0000000u); - AppendSMovLiteral(&code, 1, (20u << 16u) | 20u); - code.push_back(EncodeSop2(0x27, 2, 0, 1)); - AppendSMovLiteral(&code, 3, 0x80000000u); - AppendSMovLiteral(&code, 4, (7u << 16u) | 31u); - code.push_back(EncodeSop2(0x27, 5, 3, 4)); - AppendVMovLiteral(&code, 6, 0xf0000000u); - AppendVMovU32(&code, 7, 28); - AppendVMovU32(&code, 8, 8); - AppendVop3(&code, 0x148, 9, Vgpr(6), Vgpr(7), Vgpr(8)); - AppendStoreSgpr(&code, 2, 0); - AppendStoreSgpr(&code, 5, 1); - AppendStoreVgpr(&code, 9, 2); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 0, 0xf0000000u); + AppendSMovLiteral(&code, 1, (20u << 16u) | 20u); + code.push_back(EncodeSop2(0x27, 2, 0, 1)); + AppendSMovLiteral(&code, 3, 0x80000000u); + AppendSMovLiteral(&code, 4, (7u << 16u) | 31u); + code.push_back(EncodeSop2(0x27, 5, 3, 4)); + AppendVMovLiteral(&code, 6, 0xf0000000u); + AppendVMovU32(&code, 7, 28); + AppendVMovU32(&code, 8, 8); + AppendVop3(&code, 0x148, 9, Vgpr(6), Vgpr(7), Vgpr(8)); + AppendStoreSgpr(&code, 2, 0); + AppendStoreSgpr(&code, 5, 1); + AppendStoreVgpr(&code, 9, 2); + AppendEnd(&code); - return {"BitfieldExtractWidthPastEndEdges", - code, - {}, - {0x00000f00u, 1, 0x0000000fu}, - {O::SMovB32, O::SBfeU32, O::VMovB32, O::VBfeU32, O::BufferStoreDword, O::SEndpgm}}; + return {"BitfieldExtractWidthPastEndEdges", + code, + {}, + {0x00000f00u, 1, 0x0000000fu}, + {O::SMovB32, O::SBfeU32, O::VMovB32, O::VBfeU32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase Scalar64BitOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 0x0f0f0f0fu); - AppendSMovLiteral(&code, 1, 0x00ff00ffu); - AppendSMovLiteral(&code, 2, 0x33333333u); - AppendSMovLiteral(&code, 3, 0x0f0f0f0fu); - code.push_back(EncodeSop2(0x0f, 4, 0, 2)); - code.push_back(EncodeSop2(0x15, 6, 0, 2)); - code.push_back(EncodeSop2(0x11, 8, 0, 2)); - code.push_back(EncodeSop2(0x17, 10, 0, 2)); - code.push_back(EncodeSop2(0x13, 12, 0, 2)); - code.push_back(EncodeSop2(0x19, 14, 0, 2)); - code.push_back(EncodeSop2(0x1b, 16, 0, 2)); - code.push_back(EncodeSop2(0x1d, 18, 0, 2)); - code.push_back(EncodeSop1(0x04, 20, 0)); - code.push_back(EncodeSop2(0x1f, 22, 0, InlineU32(4))); - code.push_back(EncodeSop2(0x21, 24, 0, InlineU32(8))); - code.push_back(EncodeSop2(0x25, 26, InlineU32(36), InlineU32(4))); - code.push_back(EncodeSop2(0x21, 34, 193u, InlineU32(1))); - AppendSMovLiteral(&code, 28, 0x000c0004u); - code.push_back(EncodeSop2(0x29, 30, 0, 28)); - code.push_back(EncodeSopc(0x06, 0, 0)); - code.push_back(EncodeSop2(0x0b, 32, 0, 2)); + std::vector code; + AppendSMovLiteral(&code, 0, 0x0f0f0f0fu); + AppendSMovLiteral(&code, 1, 0x00ff00ffu); + AppendSMovLiteral(&code, 2, 0x33333333u); + AppendSMovLiteral(&code, 3, 0x0f0f0f0fu); + code.push_back(EncodeSop2(0x0f, 4, 0, 2)); + code.push_back(EncodeSop2(0x15, 6, 0, 2)); + code.push_back(EncodeSop2(0x11, 8, 0, 2)); + code.push_back(EncodeSop2(0x17, 10, 0, 2)); + code.push_back(EncodeSop2(0x13, 12, 0, 2)); + code.push_back(EncodeSop2(0x19, 14, 0, 2)); + code.push_back(EncodeSop2(0x1b, 16, 0, 2)); + code.push_back(EncodeSop2(0x1d, 18, 0, 2)); + code.push_back(EncodeSop1(0x04, 20, 0)); + code.push_back(EncodeSop2(0x1f, 22, 0, InlineU32(4))); + code.push_back(EncodeSop2(0x21, 24, 0, InlineU32(8))); + code.push_back(EncodeSop2(0x25, 26, InlineU32(36), InlineU32(4))); + code.push_back(EncodeSop2(0x21, 34, 193u, InlineU32(1))); + AppendSMovLiteral(&code, 28, 0x000c0004u); + code.push_back(EncodeSop2(0x29, 30, 0, 28)); + code.push_back(EncodeSopc(0x06, 0, 0)); + code.push_back(EncodeSop2(0x0b, 32, 0, 2)); - const u32 result_pairs[] = {4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 34, 30, 32}; - u32 out = 0; - for (auto sgpr: result_pairs) { - AppendStoreSgprPair(&code, sgpr, out); - out += 2; - } - AppendEnd(&code); + const u32 result_pairs[] = {4, 6, 8, 10, 12, 14, 16, 18, + 20, 22, 24, 26, 34, 30, 32}; + u32 out = 0; + for (auto sgpr : result_pairs) { + AppendStoreSgprPair(&code, sgpr, out); + out += 2; + } + AppendEnd(&code); - return {"Scalar64BitOps", - code, - {}, - {0x03030303u, 0x000f000fu, 0x0c0c0c0cu, 0x00f000f0u, 0x3f3f3f3fu, 0x0fff0fffu, - 0xcfcfcfcfu, 0xf0fff0ffu, 0x3c3c3c3cu, 0x0ff00ff0u, 0xfcfcfcfcu, 0xfff0fff0u, - 0xc0c0c0c0u, 0xf000f000u, 0xc3c3c3c3u, 0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, - 0xf0f0f0f0u, 0x0ff00ff0u, 0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, - 0xffffffffu, 0x7fffffffu, 0x000000f0u, 0, 0x0f0f0f0fu, 0x00ff00ffu}, - {O::SMovB32, O::SMovB64, O::SAndB64, O::SAndn2B64, O::SOrB64, O::SOrn2B64, O::SXorB64, - O::SNandB64, O::SNorB64, O::SXnorB64, O::SLshlB64, O::SLshrB64, O::SBfmB64, O::SBfeU64, - O::SCmpEqU32, O::SCselectB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"Scalar64BitOps", + code, + {}, + {0x03030303u, 0x000f000fu, 0x0c0c0c0cu, 0x00f000f0u, 0x3f3f3f3fu, + 0x0fff0fffu, 0xcfcfcfcfu, 0xf0fff0ffu, 0x3c3c3c3cu, 0x0ff00ff0u, + 0xfcfcfcfcu, 0xfff0fff0u, 0xc0c0c0c0u, 0xf000f000u, 0xc3c3c3c3u, + 0xf00ff00fu, 0x0f0f0f0fu, 0x00ff00ffu, 0xf0f0f0f0u, 0x0ff00ff0u, + 0xff0f0f0fu, 0x0000ff00u, 0xfffffff0u, 0x000000ffu, 0xffffffffu, + 0x7fffffffu, 0x000000f0u, 0, 0x0f0f0f0fu, 0x00ff00ffu}, + {O::SMovB32, O::SMovB64, O::SAndB64, O::SAndn2B64, O::SOrB64, + O::SOrn2B64, O::SXorB64, O::SNandB64, O::SNorB64, O::SXnorB64, + O::SLshlB64, O::SLshrB64, O::SBfmB64, O::SBfeU64, O::SCmpEqU32, + O::SCselectB64, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarAndn2B64SccBranch() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeSMovB32(0, InlineU32(3)), - EncodeSMovB32(1, InlineU32(0)), - EncodeSMovB32(2, InlineU32(1)), - EncodeSMovB32(3, InlineU32(0)), - EncodeSop2(0x15, 4, 0, 2), - EncodeVop1(0x01, 0, InlineU32(1)), - EncodeSopp(0x04, 1), - EncodeVop1(0x01, 0, InlineU32(7)), - }; - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code = { + EncodeSMovB32(0, InlineU32(3)), + EncodeSMovB32(1, InlineU32(0)), + EncodeSMovB32(2, InlineU32(1)), + EncodeSMovB32(3, InlineU32(0)), + EncodeSop2(0x15, 4, 0, 2), + EncodeVop1(0x01, 0, InlineU32(1)), + EncodeSopp(0x04, 1), + EncodeVop1(0x01, 0, InlineU32(7)), + }; + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - return { - "ScalarAndn2B64SccBranch", - code, - {}, - {7}, - {O::SMovB32, O::SAndn2B64, O::VMovB32, O::SCbranchScc0, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarAndn2B64SccBranch", + code, + {}, + {7}, + {O::SMovB32, O::SAndn2B64, O::VMovB32, O::SCbranchScc0, + O::BufferStoreDword, O::SEndpgm}}; } TestCase ScalarLiteral() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 0x12345678u); - code.push_back(EncodeVop1(0x01, 0, 0)); - AppendBufferStoreDword(&code, 0, 30); - AppendEnd(&code); - return {"ScalarLiteral", - code, - {}, - {0x12345678u}, - {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + std::vector code; + AppendSMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeVop1(0x01, 0, 0)); + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"ScalarLiteral", + code, + {}, + {0x12345678u}, + {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorMoves() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x00, 0, 0), - EncodeVop1(0x01, 0, InlineU32(5)), - EncodeVop1(0x01, 2, Vgpr(0)), - }; - AppendBufferStoreDword(&code, 2, 30); - AppendEnd(&code); - return {"VectorRegisterMoves", - code, - {}, - {5}, - {O::VNop, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + std::vector code = { + EncodeVop1(0x00, 0, 0), + EncodeVop1(0x01, 0, InlineU32(5)), + EncodeVop1(0x01, 2, Vgpr(0)), + }; + AppendBufferStoreDword(&code, 2, 30); + AppendEnd(&code); + return {"VectorRegisterMoves", + code, + {}, + {5}, + {O::VNop, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVop3MoveAppliesFloatSourceModifiers() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 24, 0x40000000u); - AppendVop3(&code, 0x181, 1, 24, 0, 0, 0, 0, false, 0, 0x1); - AppendStoreVgpr(&code, 1, 0); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 24, 0x40000000u); + AppendVop3(&code, 0x181, 1, 24, 0, 0, 0, 0, false, 0, 0x1); + AppendStoreVgpr(&code, 1, 0); + AppendEnd(&code); - TestCase test; - test.name = "VectorVop3MoveAppliesFloatSourceModifiers"; - test.code = code; - test.expected = {0xc0000000u}; - test.opcodes = {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.required_spirv = {"OpFNegate"}; - return test; + TestCase test; + test.name = "VectorVop3MoveAppliesFloatSourceModifiers"; + test.code = code; + test.expected = {0xc0000000u}; + test.opcodes = {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpFNegate"}; + return test; } TestCase VectorIntegerOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xfffffff8u); - code.push_back(EncodeVop1(0x01, 1, InlineU32(5))); - code.push_back(EncodeVop1(0x01, 4, InlineU32(7))); - AppendVMovLiteral(&code, 10, 0x01000001u); - code.push_back(EncodeVop1(0x01, 11, InlineU32(2))); - AppendVMovLiteral(&code, 12, 0x0f0f0f0fu); - AppendVMovLiteral(&code, 13, 0x33333333u); - AppendVMovLiteral(&code, 14, 0xfffffff0u); - code.push_back(EncodeVop1(0x01, 15, InlineU32(4))); - code.push_back(EncodeVop1(0x01, 29, InlineU32(15))); + std::vector code; + AppendVMovLiteral(&code, 0, 0xfffffff8u); + code.push_back(EncodeVop1(0x01, 1, InlineU32(5))); + code.push_back(EncodeVop1(0x01, 4, InlineU32(7))); + AppendVMovLiteral(&code, 10, 0x01000001u); + code.push_back(EncodeVop1(0x01, 11, InlineU32(2))); + AppendVMovLiteral(&code, 12, 0x0f0f0f0fu); + AppendVMovLiteral(&code, 13, 0x33333333u); + AppendVMovLiteral(&code, 14, 0xfffffff0u); + code.push_back(EncodeVop1(0x01, 15, InlineU32(4))); + code.push_back(EncodeVop1(0x01, 29, InlineU32(15))); - code.push_back(EncodeVop2(0x11, 2, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x12, 3, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x13, 5, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x14, 6, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x25, 7, Vgpr(4), 1)); - code.push_back(EncodeVop2(0x26, 8, Vgpr(4), 1)); - code.push_back(EncodeVop2(0x27, 9, InlineU32(3), 4)); - code.push_back(EncodeVop2(0x26, 35, 249, 11)); - code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6)); - code.push_back(EncodeVop2(0x27, 36, 249, 11)); - code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6)); - code.push_back(EncodeVop2(0x0b, 16, Vgpr(10), 11)); - code.push_back(EncodeVop2(0x09, 37, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x1b, 17, Vgpr(12), 13)); - code.push_back(EncodeVop2(0x1c, 18, Vgpr(12), 13)); - code.push_back(EncodeVop2(0x1d, 19, Vgpr(12), 13)); - code.push_back(EncodeVop2(0x1e, 20, Vgpr(12), 13)); - code.push_back(EncodeVop2(0x19, 21, Vgpr(11), 15)); - code.push_back(EncodeVop2(0x1a, 22, InlineU32(4), 11)); - code.push_back(EncodeVop2(0x15, 23, Vgpr(14), 15)); - code.push_back(EncodeVop2(0x16, 24, InlineU32(4), 14)); - code.push_back(EncodeVop2(0x17, 25, Vgpr(14), 15)); - code.push_back(EncodeVop2(0x18, 26, InlineU32(4), 14)); - code.push_back(EncodeVop1(0x37, 27, Vgpr(12))); - code.push_back(EncodeVop1(0x38, 28, Vgpr(29))); - code.push_back(EncodeVop1(0x3a, 30, Vgpr(11))); - code.push_back(EncodeVop1(0x39, 32, Vgpr(11))); - code.push_back(EncodeVopc(0xc2, Vgpr(4), 4)); - code.push_back(EncodeVop2(0x01, 33, InlineU32(3), 4)); - code.push_back(EncodeVop2(0x11, 34, 249, 4)); - code.push_back(EncodeVop2Sdwa(0)); + code.push_back(EncodeVop2(0x11, 2, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x12, 3, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x13, 5, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x14, 6, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x25, 7, Vgpr(4), 1)); + code.push_back(EncodeVop2(0x26, 8, Vgpr(4), 1)); + code.push_back(EncodeVop2(0x27, 9, InlineU32(3), 4)); + code.push_back(EncodeVop2(0x26, 35, 249, 11)); + code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6)); + code.push_back(EncodeVop2(0x27, 36, 249, 11)); + code.push_back(EncodeVop2Sdwa(12, 6, 0, 0, 6)); + code.push_back(EncodeVop2(0x0b, 16, Vgpr(10), 11)); + code.push_back(EncodeVop2(0x09, 37, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x1b, 17, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x1c, 18, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x1d, 19, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x1e, 20, Vgpr(12), 13)); + code.push_back(EncodeVop2(0x19, 21, Vgpr(11), 15)); + code.push_back(EncodeVop2(0x1a, 22, InlineU32(4), 11)); + code.push_back(EncodeVop2(0x15, 23, Vgpr(14), 15)); + code.push_back(EncodeVop2(0x16, 24, InlineU32(4), 14)); + code.push_back(EncodeVop2(0x17, 25, Vgpr(14), 15)); + code.push_back(EncodeVop2(0x18, 26, InlineU32(4), 14)); + code.push_back(EncodeVop1(0x37, 27, Vgpr(12))); + code.push_back(EncodeVop1(0x38, 28, Vgpr(29))); + code.push_back(EncodeVop1(0x3a, 30, Vgpr(11))); + code.push_back(EncodeVop1(0x39, 32, Vgpr(11))); + code.push_back(EncodeVopc(0xc2, Vgpr(4), 4)); + code.push_back(EncodeVop2(0x01, 33, InlineU32(3), 4)); + code.push_back(EncodeVop2(0x11, 34, 249, 4)); + code.push_back(EncodeVop2Sdwa(0)); - const u32 results[] = {2, 3, 5, 6, 7, 8, 9, 35, 36, 16, 37, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 32, 33, 34}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {2, 3, 5, 6, 7, 8, 9, 35, 36, 16, 37, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 30, 32, 33, 34}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"VectorIntegerOps", - code, - {}, - {0xfffffff8u, 5, 5, 0xfffffff8u, 12, 2, 4, - 13, 0xfffffff3u, 2, 0xffffffd8u, 0x03030303u, 0x3f3f3f3fu, 0x3c3c3c3cu, - 0xc3c3c3c3u, 32, 32, 0x0fffffffu, 0x0fffffffu, 0xffffffffu, 0xffffffffu, - 0xf0f0f0f0u, 0xf0000000u, 1, 30, 7, 0xfffffff8u}, - {O::VMovB32, O::VMinI32, O::VMaxI32, O::VMinU32, O::VMaxU32, - O::VAddNcU32, O::VSubNcU32, O::VSubrevNcU32, O::VMulU32U24, O::VMulI32I24, - O::VAndB32, O::VOrB32, O::VXorB32, O::VXnorB32, O::VLshlB32, - O::VLshlrevB32, O::VLshrB32, O::VLshrrevB32, O::VAshrI32, O::VAshrrevI32, - O::VNotB32, O::VBfrevB32, O::VFfblB32, O::VFfbhU32, O::VCmpEqU32, - O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorIntegerOps", + code, + {}, + {0xfffffff8u, 5, 5, 0xfffffff8u, 12, + 2, 4, 13, 0xfffffff3u, 2, + 0xffffffd8u, 0x03030303u, 0x3f3f3f3fu, 0x3c3c3c3cu, 0xc3c3c3c3u, + 32, 32, 0x0fffffffu, 0x0fffffffu, 0xffffffffu, + 0xffffffffu, 0xf0f0f0f0u, 0xf0000000u, 1, 30, + 7, 0xfffffff8u}, + {O::VMovB32, O::VMinI32, O::VMaxI32, O::VMinU32, + O::VMaxU32, O::VAddNcU32, O::VSubNcU32, O::VSubrevNcU32, + O::VMulU32U24, O::VMulI32I24, O::VAndB32, O::VOrB32, + O::VXorB32, O::VXnorB32, O::VLshlB32, O::VLshlrevB32, + O::VLshrB32, O::VLshrrevB32, O::VAshrI32, O::VAshrrevI32, + O::VNotB32, O::VBfrevB32, O::VFfblB32, O::VFfbhU32, + O::VCmpEqU32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase Vop2SdwaSubNcExactByte2Destination() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 11, 2); - code.push_back(0x4c1616f9u); - code.push_back(0x0686128du); - AppendStoreVgpr(&code, 11, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 11, 2); + code.push_back(0x4c1616f9u); + code.push_back(0x0686128du); + AppendStoreVgpr(&code, 11, 0); + AppendEnd(&code); - TestCase test; - test.name = "Vop2SdwaSubNcExactByte2Destination"; - test.code = code; - test.expected = {0x000b0002u}; - test.opcodes = {O::VMovB32, O::VSubNcU32, O::BufferStoreDword, O::SEndpgm}; - test.required_spirv = {"OpISub", "OpShiftLeftLogical", "OpBitwiseOr"}; - return test; + TestCase test; + test.name = "Vop2SdwaSubNcExactByte2Destination"; + test.code = code; + test.expected = {0x000b0002u}; + test.opcodes = {O::VMovB32, O::VSubNcU32, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpISub", "OpShiftLeftLogical"}; + return test; } TestCase Vop2SdwaSubNcPreservesByteAndWordDestinations() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 0, 3); - for (u32 dst = 10; dst <= 15; dst++) { - AppendVMovLiteral(&code, dst, 0xa1b2c3d4u); - } - for (u32 sel = 0; sel <= 5; sel++) { - code.push_back(EncodeVop2(0x26, 10 + sel, 249, 0)); - code.push_back(EncodeVop2Sdwa(InlineU32(20), sel, 2, 6, 6, 0, 0, 0, 0, 0, 0, 1)); - AppendStoreVgpr(&code, 10 + sel, sel); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 0, 3); + for (u32 dst = 10; dst <= 15; dst++) { + AppendVMovLiteral(&code, dst, 0xa1b2c3d4u); + } + for (u32 sel = 0; sel <= 5; sel++) { + code.push_back(EncodeVop2(0x26, 10 + sel, 249, 0)); + code.push_back( + EncodeVop2Sdwa(InlineU32(20), sel, 2, 6, 6, 0, 0, 0, 0, 0, 0, 1)); + AppendStoreVgpr(&code, 10 + sel, sel); + } + AppendEnd(&code); - return {"Vop2SdwaSubNcPreservesByteAndWordDestinations", - code, - {}, - {0xa1b2c311u, 0xa1b211d4u, 0xa111c3d4u, 0x11b2c3d4u, 0xa1b20011u, - 0x0011c3d4u}, - {O::VMovB32, O::VSubNcU32, O::BufferStoreDword, O::SEndpgm}}; + return {"Vop2SdwaSubNcPreservesByteAndWordDestinations", + code, + {}, + {0xa1b2c311u, 0xa1b211d4u, 0xa111c3d4u, 0x11b2c3d4u, 0xa1b20011u, + 0x0011c3d4u}, + {O::VMovB32, O::VSubNcU32, O::BufferStoreDword, O::SEndpgm}}; } TestCase Vop2SdwaMinU32PreservesWordDestination() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 10, 0xa1b2c3d4u); - AppendVMovU32(&code, 12, 7); - code.push_back(0x261418f9u); - code.push_back(0x0686149fu); - AppendStoreVgpr(&code, 10, 0); - AppendVMovLiteral(&code, 10, 0xa1b2c3d4u); - AppendVMovU32(&code, 12, 64); - code.push_back(0x261418f9u); - code.push_back(0x0686149fu); - AppendStoreVgpr(&code, 10, 1); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 10, 0xa1b2c3d4u); + AppendVMovU32(&code, 12, 7); + code.push_back(0x261418f9u); + code.push_back(0x0686149fu); + AppendStoreVgpr(&code, 10, 0); + AppendVMovLiteral(&code, 10, 0xa1b2c3d4u); + AppendVMovU32(&code, 12, 64); + code.push_back(0x261418f9u); + code.push_back(0x0686149fu); + AppendStoreVgpr(&code, 10, 1); + AppendEnd(&code); - TestCase test; - test.name = "Vop2SdwaMinU32PreservesWordDestination"; - test.code = code; - test.expected = {0xa1b20007u, 0xa1b2001fu}; - test.opcodes = {O::VMovB32, O::VMinU32, O::BufferStoreDword, O::SEndpgm}; - test.required_spirv = {"OpULessThan", "OpSelect", "OpBitwiseOr"}; - return test; + TestCase test; + test.name = "Vop2SdwaMinU32PreservesWordDestination"; + test.code = code; + test.expected = {0xa1b20007u, 0xa1b2001fu}; + test.opcodes = {O::VMovB32, O::VMinU32, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpULessThan", "OpSelect"}; + return test; } TestCase VectorShiftCountsMaskLowBits() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 0, 1); - AppendVMovU32(&code, 1, 32); - AppendVMovU32(&code, 2, 33); - AppendVMovLiteral(&code, 3, 0x80000000u); - code.push_back(EncodeVop2(0x19, 10, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x1a, 11, Vgpr(2), 0)); - code.push_back(EncodeVop2(0x15, 12, Vgpr(3), 1)); - code.push_back(EncodeVop2(0x16, 13, Vgpr(2), 3)); - code.push_back(EncodeVop2(0x17, 14, Vgpr(3), 1)); - code.push_back(EncodeVop2(0x18, 15, Vgpr(2), 3)); + std::vector code; + AppendVMovU32(&code, 0, 1); + AppendVMovU32(&code, 1, 32); + AppendVMovU32(&code, 2, 33); + AppendVMovLiteral(&code, 3, 0x80000000u); + code.push_back(EncodeVop2(0x19, 10, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x1a, 11, Vgpr(2), 0)); + code.push_back(EncodeVop2(0x15, 12, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x16, 13, Vgpr(2), 3)); + code.push_back(EncodeVop2(0x17, 14, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x18, 15, Vgpr(2), 3)); - for (u32 i = 0; i < 6u; i++) { - AppendStoreVgpr(&code, 10u + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 6u; i++) { + AppendStoreVgpr(&code, 10u + i, i); + } + AppendEnd(&code); - return {"VectorShiftCountsMaskLowBits", - code, - {}, - {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, - {O::VMovB32, O::VLshlB32, O::VLshlrevB32, O::VLshrB32, O::VLshrrevB32, O::VAshrI32, - O::VAshrrevI32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorShiftCountsMaskLowBits", + code, + {}, + {1, 2, 0x80000000u, 0x40000000u, 0x80000000u, 0xc0000000u}, + {O::VMovB32, O::VLshlB32, O::VLshlrevB32, O::VLshrB32, O::VLshrrevB32, + O::VAshrI32, O::VAshrrevI32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVop3IntegerOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x01, 0, InlineU32(2))); - code.push_back(EncodeVop1(0x01, 1, InlineU32(3))); - code.push_back(EncodeVop1(0x01, 2, InlineU32(4))); - AppendVMovLiteral(&code, 3, 0xfffffff8u); - code.push_back(EncodeVop1(0x01, 4, InlineU32(5))); - AppendVMovLiteral(&code, 5, 0x11223344u); - AppendVMovLiteral(&code, 6, 0x55667788u); - AppendVMovLiteral(&code, 7, 0x0f0f0f0fu); - AppendVMovLiteral(&code, 8, 0x33333333u); - AppendVMovLiteral(&code, 9, 0xaaaaaaaau); - AppendVMovLiteral(&code, 14, 0x00f00000u); - code.push_back(EncodeVop1(0x01, 15, InlineU32(8))); - AppendVMovLiteral(&code, 30, 0x00010000u); - AppendVMovLiteral(&code, 31, 0x00010000u); + std::vector code; + code.push_back(EncodeVop1(0x01, 0, InlineU32(2))); + code.push_back(EncodeVop1(0x01, 1, InlineU32(3))); + code.push_back(EncodeVop1(0x01, 2, InlineU32(4))); + AppendVMovLiteral(&code, 3, 0xfffffff8u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(5))); + AppendVMovLiteral(&code, 5, 0x11223344u); + AppendVMovLiteral(&code, 6, 0x55667788u); + AppendVMovLiteral(&code, 7, 0x0f0f0f0fu); + AppendVMovLiteral(&code, 8, 0x33333333u); + AppendVMovLiteral(&code, 9, 0xaaaaaaaau); + AppendVMovLiteral(&code, 14, 0x00f00000u); + code.push_back(EncodeVop1(0x01, 15, InlineU32(8))); + AppendVMovLiteral(&code, 30, 0x00010000u); + AppendVMovLiteral(&code, 31, 0x00010000u); - AppendVop3(&code, 0x36d, 10, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x152, 11, Vgpr(3), Vgpr(4), Vgpr(0)); - AppendVop3(&code, 0x155, 12, Vgpr(3), Vgpr(4), Vgpr(0)); - AppendVop3(&code, 0x158, 13, Vgpr(3), Vgpr(4), Vgpr(0)); - AppendVop3(&code, 0x153, 16, Vgpr(3), Vgpr(4), Vgpr(0)); - AppendVop3(&code, 0x156, 17, Vgpr(3), Vgpr(4), Vgpr(0)); - AppendVop3(&code, 0x159, 18, Vgpr(3), Vgpr(4), Vgpr(0)); - AppendVop3(&code, 0x15d, 19, Vgpr(0), Vgpr(4), Vgpr(2)); - AppendVop3(&code, 0x346, 20, Vgpr(1), Vgpr(0), Vgpr(2)); - AppendVop3(&code, 0x347, 21, Vgpr(0), Vgpr(1), Vgpr(0)); - AppendVop3(&code, 0x345, 22, Vgpr(7), Vgpr(8), Vgpr(0)); - AppendVop3(&code, 0x36f, 23, Vgpr(1), Vgpr(2), Vgpr(0)); - AppendVop3(&code, 0x371, 24, Vgpr(7), Vgpr(8), Vgpr(2)); - AppendVop3(&code, 0x372, 25, Vgpr(7), Vgpr(8), Vgpr(2)); - AppendVop3(&code, 0x178, 26, Vgpr(7), Vgpr(8), Vgpr(2)); - AppendVop3(&code, 0x148, 27, Vgpr(14), InlineU32(20), Vgpr(2)); - AppendVop3(&code, 0x149, 28, Vgpr(3), Vgpr(2), Vgpr(2)); - AppendVop3(&code, 0x14a, 29, Vgpr(8), Vgpr(7), Vgpr(9)); - AppendVop3(&code, 0x14e, 32, Vgpr(5), Vgpr(6), Vgpr(15)); - AppendVop3(&code, 0x363, 33, Vgpr(2), Vgpr(15)); - AppendVop3(&code, 0x169, 34, Vgpr(30), Vgpr(31)); - AppendVop3(&code, 0x16a, 35, Vgpr(30), Vgpr(31)); - AppendVop3(&code, 0x16b, 36, Vgpr(30), Vgpr(31)); - AppendVop3(&code, 0x16c, 44, Vgpr(3), Vgpr(30)); - AppendVop3(&code, 0x142, 37, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x143, 38, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x30f, 39, Vgpr(0), Vgpr(1)); - AppendVop3(&code, 0x310, 40, Vgpr(0), Vgpr(1)); - AppendVop3(&code, 0x319, 41, Vgpr(0), Vgpr(1)); + AppendVop3(&code, 0x36d, 10, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x152, 11, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x155, 12, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x158, 13, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x153, 16, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x156, 17, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x159, 18, Vgpr(3), Vgpr(4), Vgpr(0)); + AppendVop3(&code, 0x15d, 19, Vgpr(0), Vgpr(4), Vgpr(2)); + AppendVop3(&code, 0x346, 20, Vgpr(1), Vgpr(0), Vgpr(2)); + AppendVop3(&code, 0x347, 21, Vgpr(0), Vgpr(1), Vgpr(0)); + AppendVop3(&code, 0x345, 22, Vgpr(7), Vgpr(8), Vgpr(0)); + AppendVop3(&code, 0x36f, 23, Vgpr(1), Vgpr(2), Vgpr(0)); + AppendVop3(&code, 0x371, 24, Vgpr(7), Vgpr(8), Vgpr(2)); + AppendVop3(&code, 0x372, 25, Vgpr(7), Vgpr(8), Vgpr(2)); + AppendVop3(&code, 0x178, 26, Vgpr(7), Vgpr(8), Vgpr(2)); + AppendVop3(&code, 0x148, 27, Vgpr(14), InlineU32(20), Vgpr(2)); + AppendVop3(&code, 0x149, 28, Vgpr(3), Vgpr(2), Vgpr(2)); + AppendVop3(&code, 0x14a, 29, Vgpr(8), Vgpr(7), Vgpr(9)); + AppendVop3(&code, 0x14e, 32, Vgpr(5), Vgpr(6), Vgpr(15)); + AppendVop3(&code, 0x363, 33, Vgpr(2), Vgpr(15)); + AppendVop3(&code, 0x169, 34, Vgpr(30), Vgpr(31)); + AppendVop3(&code, 0x16a, 35, Vgpr(30), Vgpr(31)); + AppendVop3(&code, 0x16b, 36, Vgpr(30), Vgpr(31)); + AppendVop3(&code, 0x16c, 44, Vgpr(3), Vgpr(30)); + AppendVop3(&code, 0x142, 37, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x143, 38, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x30f, 39, Vgpr(0), Vgpr(1)); + AppendVop3(&code, 0x310, 40, Vgpr(0), Vgpr(1)); + AppendVop3(&code, 0x319, 41, Vgpr(0), Vgpr(1)); - const u32 results[] = {10, 11, 12, 13, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 32, 33, 34, 35, 36, 44, 37, 38, 39, 40, 41}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {10, 11, 12, 13, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, + 34, 35, 36, 44, 37, 38, 39, 40, 41}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"VectorVop3IntegerOps", - code, - {}, - {9, - 0xfffffff8u, - 5, - 2, - 2, - 0xfffffff8u, - 5, - 7, - 16, - 20, - 0x3c3c3c3eu, - 0x32, - 0x03030307u, - 0x3f3f3f3fu, - 0x3c3c3c38u, - 0x0fu, - 0x0fu, - 0x8b8b8b8bu, - 0x44556677u, - 0x00000f00u, - 0, - 1, - 0, - 0xffffffffu, - 10, - 10, - 5, - 0xffffffffu, - 1}, - {O::VMovB32, O::VAdd3U32, O::VMin3I32, O::VMax3I32, O::VMed3I32, - O::VMin3U32, O::VMax3U32, O::VMed3U32, O::VSadU32, O::VLshlAddU32, - O::VAddLshlU32, O::VXadU32, O::VLshlOrB32, O::VAndOrB32, O::VOr3B32, - O::VXor3B32, O::VBfeU32, O::VBfeI32, O::VBfiB32, O::VAlignbitB32, - O::VBfmB32, O::VMulLoU32, O::VMulHiU32, O::VMulLoI32, O::VMulHiI32, - O::VMadI32I24, O::VMadU32U24, O::VAddI32, O::VSubI32, O::VSubrevI32, - O::BufferStoreDword, O::SEndpgm}}; + return {"VectorVop3IntegerOps", + code, + {}, + {9, + 0xfffffff8u, + 5, + 2, + 2, + 0xfffffff8u, + 5, + 7, + 16, + 20, + 0x3c3c3c3eu, + 0x32, + 0x03030307u, + 0x3f3f3f3fu, + 0x3c3c3c38u, + 0x0fu, + 0x0fu, + 0x8b8b8b8bu, + 0x44556677u, + 0x00000f00u, + 0, + 1, + 0, + 0xffffffffu, + 10, + 10, + 5, + 0xffffffffu, + 1}, + {O::VMovB32, O::VAdd3U32, O::VMin3I32, O::VMax3I32, + O::VMed3I32, O::VMin3U32, O::VMax3U32, O::VMed3U32, + O::VSadU32, O::VLshlAddU32, O::VAddLshlU32, O::VXadU32, + O::VLshlOrB32, O::VAndOrB32, O::VOr3B32, O::VXor3B32, + O::VBfeU32, O::VBfeI32, O::VBfiB32, O::VAlignbitB32, + O::VBfmB32, O::VMulLoU32, O::VMulHiU32, O::VMulLoI32, + O::VMulHiI32, O::VMadI32I24, O::VMadU32U24, O::VAddI32, + O::VSubI32, O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorBfeI32ArithmeticShiftMasksField() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x80000000u); - AppendVMovLiteral(&code, 1, 0xfffffff8u); - AppendVMovU32(&code, 2, 31); - AppendVMovU32(&code, 3, 1); - AppendVMovU32(&code, 4, 3); - AppendVMovU32(&code, 5, 4); - AppendVop3(&code, 0x149, 10, Vgpr(0), Vgpr(2), Vgpr(3)); - AppendVop3(&code, 0x149, 11, Vgpr(1), Vgpr(4), Vgpr(5)); - AppendStoreVgpr(&code, 10, 0); - AppendStoreVgpr(&code, 11, 1); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x80000000u); + AppendVMovLiteral(&code, 1, 0xfffffff8u); + AppendVMovU32(&code, 2, 31); + AppendVMovU32(&code, 3, 1); + AppendVMovU32(&code, 4, 3); + AppendVMovU32(&code, 5, 4); + AppendVop3(&code, 0x149, 10, Vgpr(0), Vgpr(2), Vgpr(3)); + AppendVop3(&code, 0x149, 11, Vgpr(1), Vgpr(4), Vgpr(5)); + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendEnd(&code); - return {"VectorBfeI32ArithmeticShiftMasksField", - code, - {}, - {1, 0x0fu}, - {O::VMovB32, O::VBfeI32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorBfeI32ArithmeticShiftMasksField", + code, + {}, + {1, 0x0fu}, + {O::VMovB32, O::VBfeI32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorCarryAndBitCountOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xffffffffu); - code.push_back(EncodeVop1(0x01, 1, InlineU32(0))); - AppendVMovLiteral(&code, 3, 0x0000f0f0u); - code.push_back(EncodeVop1(0x01, 4, InlineU32(5))); + std::vector code; + AppendVMovLiteral(&code, 0, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 1, InlineU32(0))); + AppendVMovLiteral(&code, 3, 0x0000f0f0u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(5))); - code.push_back(EncodeVopc(0xc7, Vgpr(1), 1)); - code.push_back(EncodeVop2(0x28, 2, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x22, 5, Vgpr(3), 4)); - AppendVMovLiteral(&code, 6, 0xffffffffu); - code.push_back(EncodeVop2(0x23, 7, Vgpr(6), 4)); - code.push_back(EncodeVop2(0x24, 8, Vgpr(6), 4)); + code.push_back(EncodeVopc(0xc7, Vgpr(1), 1)); + code.push_back(EncodeVop2(0x28, 2, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x22, 5, Vgpr(3), 4)); + AppendVMovLiteral(&code, 6, 0xffffffffu); + code.push_back(EncodeVop2(0x23, 7, Vgpr(6), 4)); + code.push_back(EncodeVop2(0x24, 8, Vgpr(6), 4)); - AppendStoreVgpr(&code, 2, 0); - AppendStoreSgpr(&code, 106, 1); - AppendStoreVgpr(&code, 5, 2); - AppendStoreVgpr(&code, 7, 3); - AppendStoreVgpr(&code, 8, 4); - AppendEnd(&code); + AppendStoreVgpr(&code, 2, 0); + AppendStoreSgpr(&code, 106, 1); + AppendStoreVgpr(&code, 5, 2); + AppendStoreVgpr(&code, 7, 3); + AppendStoreVgpr(&code, 8, 4); + AppendEnd(&code); - return {"VectorCarryAndBitCountOps", - code, - {}, - {0, 1, 13, 5, 5}, - {O::VMovB32, O::VCmpTU32, O::VAddcU32, O::VBcntU32B32, O::VMbcntLoU32B32, - O::VMbcntHiU32B32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorCarryAndBitCountOps", + code, + {}, + {0, 1, 13, 5, 5}, + {O::VMovB32, O::VCmpTU32, O::VAddcU32, O::VBcntU32B32, + O::VMbcntLoU32B32, O::VMbcntHiU32B32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase VectorMbcntUsesThreadMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVop2(0x1a, 3, InlineU32(2), 1), - }; - AppendVMovLiteral(&code, 2, 0xffffffffu); - code.push_back(EncodeVop1(0x01, 4, InlineU32(0))); - code.push_back(EncodeVop2(0x23, 5, Vgpr(2), 4)); - code.push_back(EncodeVop2(0x24, 6, Vgpr(2), 5)); - AppendBufferStoreDword(&code, 6, 3); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + }; + AppendVMovLiteral(&code, 2, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 4, InlineU32(0))); + code.push_back(EncodeVop2(0x23, 5, Vgpr(2), 4)); + code.push_back(EncodeVop2(0x24, 6, Vgpr(2), 5)); + AppendBufferStoreDword(&code, 6, 3); + AppendEnd(&code); - TestCase test; - test.name = "VectorMbcntUsesThreadMask"; - test.code = code; - test.expected = {0, 1, 2, 3, 0, 1, 2, 3}; - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VMbcntLoU32B32, - O::VMbcntHiU32B32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "VectorMbcntUsesThreadMask"; + test.code = code; + test.expected = {0, 1, 2, 3, 0, 1, 2, 3}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VMbcntLoU32B32, O::VMbcntHiU32B32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase VectorAddcUsesPerLaneCarryIn() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVopc(0xc7, Vgpr(0), 0), - EncodeVop1(0x01, 2, InlineU32(0)), - EncodeVop2(0x28, 3, Vgpr(2), 2), - EncodeVop2(0x1a, 4, InlineU32(2), 1), - }; - AppendBufferStoreDword(&code, 3, 4); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVopc(0xc7, Vgpr(0), 0), + EncodeVop1(0x01, 2, InlineU32(0)), + EncodeVop2(0x28, 3, Vgpr(2), 2), + EncodeVop2(0x1a, 4, InlineU32(2), 1), + }; + AppendBufferStoreDword(&code, 3, 4); + AppendEnd(&code); - TestCase test; - test.name = "VectorAddcUsesPerLaneCarryIn"; - test.code = code; - test.expected = std::vector(8, 1); - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpTU32, - O::VAddcU32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "VectorAddcUsesPerLaneCarryIn"; + test.code = code; + test.expected = std::vector(8, 1); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpTU32, + O::VAddcU32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase VectorAddcWritesPerLaneCarryOut() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVopc(0xc0, Vgpr(0), 0), - }; - AppendVMovLiteral(&code, 2, 0xffffffffu); - code.push_back(EncodeVop1(0x01, 3, InlineU32(1))); - code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 3)); - code.push_back(EncodeVop1(0x01, 6, 106)); - code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 1)); - AppendBufferStoreDword(&code, 6, 4); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVopc(0xc0, Vgpr(0), 0), + }; + AppendVMovLiteral(&code, 2, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 3, InlineU32(1))); + code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 3)); + code.push_back(EncodeVop1(0x01, 6, 106)); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 1)); + AppendBufferStoreDword(&code, 6, 4); + AppendEnd(&code); - TestCase test; - test.name = "VectorAddcWritesPerLaneCarryOut"; - test.code = code; - test.expected = std::vector(8, 0x0fu); - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, - O::VAddcU32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "VectorAddcWritesPerLaneCarryOut"; + test.code = code; + test.expected = std::vector(8, 0x0fu); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, + O::VAddcU32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase VectorVop3BCarryOutWritesSgprMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 1, 0xffffffffu); - AppendVMovU32(&code, 2, 1); - AppendVMovU32(&code, 3, 0); + std::vector code; + AppendVMovLiteral(&code, 1, 0xffffffffu); + AppendVMovU32(&code, 2, 1); + AppendVMovU32(&code, 3, 0); - AppendVop3B(&code, 0x30fu, 10, 0, Vgpr(1), Vgpr(2)); - AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 0); - AppendVop3B(&code, 0x310u, 11, 0, Vgpr(3), Vgpr(2)); - AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 4); - AppendVop3B(&code, 0x319u, 12, 0, Vgpr(2), Vgpr(3)); - AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 8); - AppendEnd(&code); + AppendVop3B(&code, 0x30fu, 10, 0, Vgpr(1), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 0); + AppendVop3B(&code, 0x310u, 11, 0, Vgpr(3), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 4); + AppendVop3B(&code, 0x319u, 12, 0, Vgpr(2), Vgpr(3)); + AppendStoreSgprAtLaneDwordOffset(&code, 0, 0, 8); + AppendEnd(&code); - TestCase test; - test.name = "VectorVop3BCarryOutWritesSgprMask"; - test.code = code; - test.expected = std::vector(12, 0x0fu); - test.opcodes = {O::VMovB32, O::VAddI32, O::VSubI32, - O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorVop3BCarryOutWritesSgprMask"; + test.code = code; + test.expected = std::vector(12, 0x0fu); + test.opcodes = {O::VMovB32, O::VAddI32, O::VSubI32, + O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; } TestCase VectorVop3BCarryOutUsesEncodedSdst() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 1, 0xffffffffu); - AppendVMovU32(&code, 2, 1); - AppendVMovU32(&code, 3, 0); + std::vector code; + AppendVMovLiteral(&code, 1, 0xffffffffu); + AppendVMovU32(&code, 2, 1); + AppendVMovU32(&code, 3, 0); - AppendVop3B(&code, 0x30fu, 10, 20, Vgpr(1), Vgpr(2)); - AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0); - AppendVop3B(&code, 0x310u, 11, 22, Vgpr(3), Vgpr(2)); - AppendStoreSgprAtLaneDwordOffset(&code, 22, 0, 4); - AppendVop3B(&code, 0x319u, 12, 24, Vgpr(2), Vgpr(3)); - AppendStoreSgprAtLaneDwordOffset(&code, 24, 0, 8); - AppendEnd(&code); + AppendVop3B(&code, 0x30fu, 10, 20, Vgpr(1), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 0); + AppendVop3B(&code, 0x310u, 11, 22, Vgpr(3), Vgpr(2)); + AppendStoreSgprAtLaneDwordOffset(&code, 22, 0, 4); + AppendVop3B(&code, 0x319u, 12, 24, Vgpr(2), Vgpr(3)); + AppendStoreSgprAtLaneDwordOffset(&code, 24, 0, 8); + AppendEnd(&code); - TestCase test; - test.name = "VectorVop3BCarryOutUsesEncodedSdst"; - test.code = code; - test.expected = std::vector(12, 0x0fu); - test.opcodes = {O::VMovB32, O::VAddI32, O::VSubI32, - O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorVop3BCarryOutUsesEncodedSdst"; + test.code = code; + test.expected = std::vector(12, 0x0fu); + test.opcodes = {O::VMovB32, O::VAddI32, O::VSubI32, + O::VSubrevI32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; } TestCase VectorVop3BSubCoU32UsesRdna2Opcode310() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - AppendVMovU32(&code, 2, 1); - AppendVMovU32(&code, 3, 1); - AppendVMovLiteral(&code, 4, 0xffffffffu); - AppendVMovLiteral(&code, 5, 0x80000000u); + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovU32(&code, 2, 1); + AppendVMovU32(&code, 3, 1); + AppendVMovLiteral(&code, 4, 0xffffffffu); + AppendVMovLiteral(&code, 5, 0x80000000u); - code.push_back(EncodeVopc(0xc2, InlineU32(1), 0)); - code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(3))); - code.push_back(EncodeVopc(0xc2, InlineU32(2), 0)); - code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(4))); - code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); - code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(5))); - code.push_back(EncodeVop2(0x01, 2, Vgpr(2), Vgpr(4))); + code.push_back(EncodeVopc(0xc2, InlineU32(1), 0)); + code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(3))); + code.push_back(EncodeVopc(0xc2, InlineU32(2), 0)); + code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(4))); + code.push_back(EncodeVopc(0xc2, InlineU32(3), 0)); + code.push_back(EncodeVop2(0x01, 1, Vgpr(1), Vgpr(5))); + code.push_back(EncodeVop2(0x01, 2, Vgpr(2), Vgpr(4))); - AppendVop3B(&code, 0x310u, 10, 20, Vgpr(1), Vgpr(2)); - AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0); - AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 4); - AppendEnd(&code); + AppendVop3B(&code, 0x310u, 10, 20, Vgpr(1), Vgpr(2)); + AppendStoreVgprAtLaneDwordOffset(&code, 10, 0, 0); + AppendStoreSgprAtLaneDwordOffset(&code, 20, 0, 4); + AppendEnd(&code); - TestCase test; - test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310"; - test.code = code; - test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9}; - test.opcodes = {O::VMovB32, O::VCmpEqU32, O::VCndmaskB32, - O::VSubI32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorVop3BSubCoU32UsesRdna2Opcode310"; + test.code = code; + test.expected = {0xffffffffu, 0, 0xfffffffeu, 0x80000001u, 9, 9, 9, 9}; + test.opcodes = {O::VMovB32, O::VCmpEqU32, O::VCndmaskB32, + O::VSubI32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + return test; } TestCase VectorMadU64U32UnsignedCarryOut() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 1, 0xffffffffu); - AppendVMovU32(&code, 2, 2); - AppendVMovU32(&code, 4, 2); - AppendVMovLiteral(&code, 5, 0xffffffffu); - AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4)); - AppendStoreVgpr(&code, 10, 0); - AppendStoreVgpr(&code, 11, 1); - AppendStoreSgprPair(&code, 20, 2); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 1, 0xffffffffu); + AppendVMovU32(&code, 2, 2); + AppendVMovU32(&code, 4, 2); + AppendVMovLiteral(&code, 5, 0xffffffffu); + AppendVop3B(&code, 0x176u, 10, 20, Vgpr(1), Vgpr(2), Vgpr(4)); + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendStoreSgprPair(&code, 20, 2); + AppendEnd(&code); - return {"VectorMadU64U32UnsignedCarryOut", - code, - {}, - {0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u}, - {O::VMovB32, O::VMadU64U32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorMadU64U32UnsignedCarryOut", + code, + {}, + {0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u}, + {O::VMovB32, O::VMadU64U32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorLaneAndPackedOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x12345678u); - code.push_back(EncodeVop1(0x01, 1, InlineU32(0))); - AppendVMovLiteral(&code, 4, 0x40003c00u); - AppendVMovLiteral(&code, 5, 0x44004200u); - AppendVMovLiteral(&code, 6, 0x3c003c00u); - AppendVMovLiteral(&code, 7, 0x3f800000u); - AppendVMovLiteral(&code, 8, 0x40000000u); - AppendVMovLiteral(&code, 9, 0x12345678u); - AppendVMovLiteral(&code, 10, 0xabcdef01u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeVop1(0x01, 1, InlineU32(0))); + AppendVMovLiteral(&code, 4, 0x40003c00u); + AppendVMovLiteral(&code, 5, 0x44004200u); + AppendVMovLiteral(&code, 6, 0x3c003c00u); + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendVMovLiteral(&code, 8, 0x40000000u); + AppendVMovLiteral(&code, 9, 0x12345678u); + AppendVMovLiteral(&code, 10, 0xabcdef01u); - code.push_back(EncodeVop1(0x02, 0, Vgpr(0))); - AppendVop3(&code, 0x360, 1, Vgpr(0), InlineU32(0)); - AppendVop3(&code, 0x361, 1, 0, InlineU32(0)); - AppendVop3(&code, 0x377, 2, Vgpr(0), InlineU32(0), InlineU32(0)); - code.push_back(EncodeVop2(0x2f, 3, Vgpr(7), 8)); - AppendVop3p(&code, 0x0f, 11, Vgpr(4), Vgpr(5), 0, 0x3); - AppendVop3p(&code, 0x10, 12, Vgpr(4), Vgpr(5), 0, 0x3); - AppendVop3p(&code, 0x11, 13, Vgpr(4), Vgpr(5), 0, 0x3); - AppendVop3p(&code, 0x12, 14, Vgpr(4), Vgpr(5), 0, 0x3); - AppendVop3p(&code, 0x0e, 15, Vgpr(4), Vgpr(5), Vgpr(6), 0x7); - AppendVop3(&code, 0x362, 16, Vgpr(7), InlineU32(1)); - AppendVop3(&code, 0x36a, 17, Vgpr(9), Vgpr(10)); + code.push_back(EncodeVop1(0x02, 0, Vgpr(0))); + AppendVop3(&code, 0x360, 1, Vgpr(0), InlineU32(0)); + AppendVop3(&code, 0x361, 1, 0, InlineU32(0)); + AppendVop3(&code, 0x377, 2, Vgpr(0), InlineU32(0), InlineU32(0)); + code.push_back(EncodeVop2(0x2f, 3, Vgpr(7), 8)); + AppendVop3p(&code, 0x0f, 11, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x10, 12, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x11, 13, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x12, 14, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x0e, 15, Vgpr(4), Vgpr(5), Vgpr(6), 0x7); + AppendVop3(&code, 0x362, 16, Vgpr(7), InlineU32(1)); + AppendVop3(&code, 0x36a, 17, Vgpr(9), Vgpr(10)); - AppendStoreSgpr(&code, 0, 0); - AppendStoreSgpr(&code, 1, 1); - const u32 results[] = {1, 2, 3, 11, 12, 13, 14, 15, 16, 17}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i + 2u); - } - AppendEnd(&code); + AppendStoreSgpr(&code, 0, 0); + AppendStoreSgpr(&code, 1, 1); + const u32 results[] = {1, 2, 3, 11, 12, 13, 14, 15, 16, 17}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i + 2u); + } + AppendEnd(&code); - return {"VectorLaneAndPackedOps", - code, - {}, - {0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u, 0x46004400u, - 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u, 0x40000000u, 0xef015678u}, - {O::VMovB32, O::VReadfirstlaneB32, O::VReadlaneB32, O::VWritelaneB32, O::VPermlane16B32, - O::VCvtPkrtzF16F32, O::VPkAddF16, O::VPkMulF16, O::VPkMinF16, O::VPkMaxF16, - O::VPkFmaF16, O::VLdexpF32, O::VCvtPkU16U32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorLaneAndPackedOps", + code, + {}, + {0x12345678u, 0x12345678u, 0x12345678u, 0x12345678u, 0x40003c00u, + 0x46004400u, 0x48004200u, 0x40003c00u, 0x44004200u, 0x48804400u, + 0x40000000u, 0xef015678u}, + {O::VMovB32, O::VReadfirstlaneB32, O::VReadlaneB32, O::VWritelaneB32, + O::VPermlane16B32, O::VCvtPkrtzF16F32, O::VPkAddF16, O::VPkMulF16, + O::VPkMinF16, O::VPkMaxF16, O::VPkFmaF16, O::VLdexpF32, + O::VCvtPkU16U32, O::BufferStoreDword, O::SEndpgm}}; } TestCase CvtPkU8F32PacksSelectedByte() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - // RDNA2 V_CVT_PK_U8_F32: uint8(S0) replaces byte S1[1:0] in S2. - AppendVMovLiteral(&code, 0, 0x414c0000u); // 12.75f truncates to 12. - AppendVMovLiteral(&code, 1, 0x43960000u); // 300.0f saturates to 255. - AppendVMovLiteral(&code, 2, 0xbf800000u); // Negative values saturate to zero. - AppendVMovLiteral(&code, 3, 0x7fc00000u); // NaN saturates to zero. - AppendVMovLiteral(&code, 4, 0x11223344u); + std::vector code; + // RDNA2 V_CVT_PK_U8_F32: uint8(S0) replaces byte S1[1:0] in S2. + AppendVMovLiteral(&code, 0, 0x414c0000u); // 12.75f truncates to 12. + AppendVMovLiteral(&code, 1, 0x43960000u); // 300.0f saturates to 255. + AppendVMovLiteral(&code, 2, 0xbf800000u); // Negative values saturate to zero. + AppendVMovLiteral(&code, 3, 0x7fc00000u); // NaN saturates to zero. + AppendVMovLiteral(&code, 4, 0x11223344u); - AppendVop3(&code, 0x15eu, 10, Vgpr(0), InlineU32(0), Vgpr(4)); - AppendVop3(&code, 0x15eu, 11, Vgpr(1), InlineU32(5), Vgpr(4)); - AppendVop3(&code, 0x15eu, 12, Vgpr(2), InlineU32(2), Vgpr(4)); - AppendVop3(&code, 0x15eu, 13, Vgpr(3), InlineU32(3), Vgpr(4)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + AppendVop3(&code, 0x15eu, 10, Vgpr(0), InlineU32(0), Vgpr(4)); + AppendVop3(&code, 0x15eu, 11, Vgpr(1), InlineU32(5), Vgpr(4)); + AppendVop3(&code, 0x15eu, 12, Vgpr(2), InlineU32(2), Vgpr(4)); + AppendVop3(&code, 0x15eu, 13, Vgpr(3), InlineU32(3), Vgpr(4)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"CvtPkU8F32PacksSelectedByte", - code, - {}, - {0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u}, - {O::VMovB32, O::VCvtPkU8F32, O::BufferStoreDword, O::SEndpgm}}; + return {"CvtPkU8F32PacksSelectedByte", + code, + {}, + {0x1122330cu, 0x1122ff44u, 0x11003344u, 0x00223344u}, + {O::VMovB32, O::VCvtPkU8F32, O::BufferStoreDword, O::SEndpgm}}; } TestCase CvtPkrtzF16F32SubnormalRoundsTowardZero() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x33c00000u); - AppendVMovLiteral(&code, 1, 0xb3c00000u); - code.push_back(EncodeVop2(0x2f, 2, Vgpr(0), 1)); - AppendStoreVgpr(&code, 2, 0); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x33c00000u); + AppendVMovLiteral(&code, 1, 0xb3c00000u); + code.push_back(EncodeVop2(0x2f, 2, Vgpr(0), 1)); + AppendStoreVgpr(&code, 2, 0); + AppendEnd(&code); - return {"CvtPkrtzF16F32SubnormalRoundsTowardZero", - code, - {}, - {0x80010001u}, - {O::VMovB32, O::VCvtPkrtzF16F32, O::BufferStoreDword, O::SEndpgm}}; + return {"CvtPkrtzF16F32SubnormalRoundsTowardZero", + code, + {}, + {0x80010001u}, + {O::VMovB32, O::VCvtPkrtzF16F32, O::BufferStoreDword, O::SEndpgm}}; } TestCase PackedMinMaxF16NanAndSignedZeroEdges() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x80004000u); - AppendVMovLiteral(&code, 1, 0x00007e00u); - AppendVMovLiteral(&code, 2, 0x00004000u); - AppendVMovLiteral(&code, 3, 0x80007e00u); - AppendVMovLiteral(&code, 4, 0x40007d01u); - AppendVMovLiteral(&code, 5, 0x7d014000u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x80004000u); + AppendVMovLiteral(&code, 1, 0x00007e00u); + AppendVMovLiteral(&code, 2, 0x00004000u); + AppendVMovLiteral(&code, 3, 0x80007e00u); + AppendVMovLiteral(&code, 4, 0x40007d01u); + AppendVMovLiteral(&code, 5, 0x7d014000u); - AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3); - AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3); - AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3); - AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x11, 10, Vgpr(0), Vgpr(1), 0, 0x3); + AppendVop3p(&code, 0x12, 11, Vgpr(2), Vgpr(3), 0, 0x3); + AppendVop3p(&code, 0x11, 12, Vgpr(4), Vgpr(5), 0, 0x3); + AppendVop3p(&code, 0x12, 13, Vgpr(4), Vgpr(5), 0, 0x3); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"PackedMinMaxF16NanAndSignedZeroEdges", - code, - {}, - {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u}, - {O::VMovB32, O::VPkMinF16, O::VPkMaxF16, O::BufferStoreDword, O::SEndpgm}}; + return {"PackedMinMaxF16NanAndSignedZeroEdges", + code, + {}, + {0x80004000u, 0x00004000u, 0x7f017f01u, 0x7f017f01u}, + {O::VMovB32, O::VPkMinF16, O::VPkMaxF16, O::BufferStoreDword, + O::SEndpgm}}; } TestCase VectorMinMaxF16Ops() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xaaaa4000u); // low=2.0h - AppendVMovLiteral(&code, 1, 0xbbbb3c00u); // low=1.0h - AppendVMovLiteral(&code, 2, 0x12345678u); - AppendVMovLiteral(&code, 3, 0x87654321u); - AppendVMovLiteral(&code, 4, 0x5555aaaau); - AppendVMovLiteral(&code, 5, 0x6666bbbbu); - AppendVMovLiteral(&code, 6, 0x7777bbbbu); - AppendVMovLiteral(&code, 7, 0x8888bbbbu); - AppendVMovLiteral(&code, 8, 0x9999bbbbu); - AppendVMovLiteral(&code, 9, 0x99990000u); - code.push_back(EncodeVop2(0x32, 5, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x33, 6, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x34, 7, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x35, 4, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x39, 2, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x3a, 3, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x35, 9, 249, 1)); - code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4)); - code.push_back(0x660000f9u); - code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4)); - AppendStoreVgpr(&code, 5, 0); - AppendStoreVgpr(&code, 6, 1); - AppendStoreVgpr(&code, 7, 2); - AppendStoreVgpr(&code, 0, 3); - AppendStoreVgpr(&code, 4, 4); - AppendStoreVgpr(&code, 2, 5); - AppendStoreVgpr(&code, 3, 6); - AppendStoreVgpr(&code, 9, 7); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0xaaaa4000u); // low=2.0h + AppendVMovLiteral(&code, 1, 0xbbbb3c00u); // low=1.0h + AppendVMovLiteral(&code, 2, 0x12345678u); + AppendVMovLiteral(&code, 3, 0x87654321u); + AppendVMovLiteral(&code, 4, 0x5555aaaau); + AppendVMovLiteral(&code, 5, 0x6666bbbbu); + AppendVMovLiteral(&code, 6, 0x7777bbbbu); + AppendVMovLiteral(&code, 7, 0x8888bbbbu); + AppendVMovLiteral(&code, 8, 0x9999bbbbu); + AppendVMovLiteral(&code, 9, 0x99990000u); + code.push_back(EncodeVop2(0x32, 5, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x33, 6, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x34, 7, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x35, 4, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x39, 2, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x3a, 3, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x35, 9, 249, 1)); + code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4)); + code.push_back(0x660000f9u); + code.push_back(EncodeVop2Sdwa(0, 4, 2, 4, 4)); + AppendStoreVgpr(&code, 5, 0); + AppendStoreVgpr(&code, 6, 1); + AppendStoreVgpr(&code, 7, 2); + AppendStoreVgpr(&code, 0, 3); + AppendStoreVgpr(&code, 4, 4); + AppendStoreVgpr(&code, 2, 5); + AppendStoreVgpr(&code, 3, 6); + AppendStoreVgpr(&code, 9, 7); + AppendEnd(&code); - return {"VectorMinMaxF16Ops", - code, - {}, - {0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u, 0x12344000u, - 0x87653c00u, 0x99994000u}, - {O::VMovB32, O::VAddF16, O::VSubF16, O::VSubrevF16, O::VMulF16, O::VMaxF16, O::VMinF16, - O::BufferStoreDword, O::SEndpgm}}; + return {"VectorMinMaxF16Ops", + code, + {}, + {0x66664200u, 0x77773c00u, 0x8888bc00u, 0xaaaa0000u, 0x55554000u, + 0x12344000u, 0x87653c00u, 0x99994000u}, + {O::VMovB32, O::VAddF16, O::VSubF16, O::VSubrevF16, O::VMulF16, + O::VMaxF16, O::VMinF16, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorCvtU16F16Sdwa() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x45004200u); // low=3.0h, high=5.0h - AppendVMovLiteral(&code, 1, 0xbc007c00u); // low=+inf, high=-1.0h - AppendVMovLiteral(&code, 2, 0x00007e00u); // low=qNaN - AppendVMovLiteral(&code, 10, 0x12345678u); - AppendVMovLiteral(&code, 11, 0x87654321u); - AppendVMovLiteral(&code, 12, 0xabcd1111u); - AppendVMovLiteral(&code, 13, 0x77772222u); - AppendVMovLiteral(&code, 14, 0x5555aaaau); - code.push_back(EncodeVop1(0x52, 10, 249)); - code.push_back(EncodeVop1Sdwa(0, 4, 2, 4)); - code.push_back(EncodeVop1(0x52, 11, 249)); - code.push_back(EncodeVop1Sdwa(0, 5, 2, 5)); - code.push_back(EncodeVop1(0x52, 12, 249)); - code.push_back(EncodeVop1Sdwa(1, 4, 2, 4)); - code.push_back(EncodeVop1(0x52, 13, 249)); - code.push_back(EncodeVop1Sdwa(1, 4, 2, 5)); - code.push_back(EncodeVop1(0x52, 14, 249)); - code.push_back(EncodeVop1Sdwa(2, 4, 2, 4)); - for (u32 i = 0; i < 5; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x45004200u); // low=3.0h, high=5.0h + AppendVMovLiteral(&code, 1, 0xbc007c00u); // low=+inf, high=-1.0h + AppendVMovLiteral(&code, 2, 0x00007e00u); // low=qNaN + AppendVMovLiteral(&code, 10, 0x12345678u); + AppendVMovLiteral(&code, 11, 0x87654321u); + AppendVMovLiteral(&code, 12, 0xabcd1111u); + AppendVMovLiteral(&code, 13, 0x77772222u); + AppendVMovLiteral(&code, 14, 0x5555aaaau); + code.push_back(EncodeVop1(0x52, 10, 249)); + code.push_back(EncodeVop1Sdwa(0, 4, 2, 4)); + code.push_back(EncodeVop1(0x52, 11, 249)); + code.push_back(EncodeVop1Sdwa(0, 5, 2, 5)); + code.push_back(EncodeVop1(0x52, 12, 249)); + code.push_back(EncodeVop1Sdwa(1, 4, 2, 4)); + code.push_back(EncodeVop1(0x52, 13, 249)); + code.push_back(EncodeVop1Sdwa(1, 4, 2, 5)); + code.push_back(EncodeVop1(0x52, 14, 249)); + code.push_back(EncodeVop1Sdwa(2, 4, 2, 4)); + for (u32 i = 0; i < 5; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"VectorCvtU16F16Sdwa", - code, - {}, - {0x12340003u, 0x00054321u, 0xabcdffffu, 0x77770000u, 0x55550000u}, - {O::VMovB32, O::VCvtU16F16, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorCvtU16F16Sdwa", + code, + {}, + {0x12340003u, 0x00054321u, 0xabcdffffu, 0x77770000u, 0x55550000u}, + {O::VMovB32, O::VCvtU16F16, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorMinMaxMed3F16Ops() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x40003c00u); // low=1.0h, high=2.0h - AppendVMovLiteral(&code, 1, 0x44004200u); // low=3.0h, high=4.0h - AppendVMovLiteral(&code, 2, 0xc8004000u); // low=2.0h, high=-8.0h - AppendVMovLiteral(&code, 3, 0x00007e00u); // low=qNaN - AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h - AppendVMovLiteral(&code, 5, 0x00003c00u); // low=1.0h - AppendVMovLiteral(&code, 10, 0xaaaa5555u); - AppendVMovLiteral(&code, 11, 0x12345678u); - AppendVMovLiteral(&code, 12, 0x77772222u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x40003c00u); // low=1.0h, high=2.0h + AppendVMovLiteral(&code, 1, 0x44004200u); // low=3.0h, high=4.0h + AppendVMovLiteral(&code, 2, 0xc8004000u); // low=2.0h, high=-8.0h + AppendVMovLiteral(&code, 3, 0x00007e00u); // low=qNaN + AppendVMovLiteral(&code, 4, 0x00004000u); // low=2.0h + AppendVMovLiteral(&code, 5, 0x00003c00u); // low=1.0h + AppendVMovLiteral(&code, 10, 0xaaaa5555u); + AppendVMovLiteral(&code, 11, 0x12345678u); + AppendVMovLiteral(&code, 12, 0x77772222u); - AppendVop3(&code, 0x351, 10, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x354, 11, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0xf, false, 0, 0x4); - AppendVop3(&code, 0x357, 12, Vgpr(3), Vgpr(4), Vgpr(5)); - for (u32 i = 0; i < 3; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + AppendVop3(&code, 0x351, 10, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x354, 11, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0xf, false, 0, + 0x4); + AppendVop3(&code, 0x357, 12, Vgpr(3), Vgpr(4), Vgpr(5)); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"VectorMinMaxMed3F16Ops", - code, - {}, - {0xaaaa3c00u, 0x48005678u, 0x77773c00u}, - {O::VMovB32, O::VMin3F16, O::VMax3F16, O::VMed3F16, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorMinMaxMed3F16Ops", + code, + {}, + {0xaaaa3c00u, 0x48005678u, 0x77773c00u}, + {O::VMovB32, O::VMin3F16, O::VMax3F16, O::VMed3F16, + O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorSpecialF16Ops() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xc000fc00u); // low=-inf, high=-2.0h - AppendVMovLiteral(&code, 1, 0x00004400u); // low=+4.0h - AppendVMovLiteral(&code, 2, 0x00003c00u); // low=+1.0h - AppendVMovLiteral(&code, 3, 0x0000bc00u); // low=-1.0h - AppendVMovLiteral(&code, 4, 0x0000fc00u); // low=-inf - AppendVMovLiteral(&code, 11, 0x12345678u); + std::vector code; + AppendVMovLiteral(&code, 0, 0xc000fc00u); // low=-inf, high=-2.0h + AppendVMovLiteral(&code, 1, 0x00004400u); // low=+4.0h + AppendVMovLiteral(&code, 2, 0x00003c00u); // low=+1.0h + AppendVMovLiteral(&code, 3, 0x0000bc00u); // low=-1.0h + AppendVMovLiteral(&code, 4, 0x0000fc00u); // low=-inf + AppendVMovLiteral(&code, 11, 0x12345678u); - code.push_back(EncodeVop1(0x54, 10, Vgpr(0))); - code.push_back(EncodeVop1(0x54, 11, 249)); - code.push_back(EncodeVop1Sdwa(0, 5, 2, 5)); - code.push_back(EncodeVop1(0x56, 12, Vgpr(1))); - code.push_back(EncodeVop1(0x57, 13, Vgpr(2))); - code.push_back(EncodeVop1(0x57, 14, Vgpr(3))); - code.push_back(EncodeVop1(0x58, 15, Vgpr(4))); - for (u32 i = 0; i < 6; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + code.push_back(EncodeVop1(0x54, 10, Vgpr(0))); + code.push_back(EncodeVop1(0x54, 11, 249)); + code.push_back(EncodeVop1Sdwa(0, 5, 2, 5)); + code.push_back(EncodeVop1(0x56, 12, Vgpr(1))); + code.push_back(EncodeVop1(0x57, 13, Vgpr(2))); + code.push_back(EncodeVop1(0x57, 14, Vgpr(3))); + code.push_back(EncodeVop1(0x58, 15, Vgpr(4))); + for (u32 i = 0; i < 6; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"VectorSpecialF16Ops", - code, - {}, - {0x00008000u, 0xb8005678u, 0x00003800u, 0x00000000u, 0x0000fe00u, 0x00000000u}, - {O::VMovB32, O::VRcpF16, O::VRsqF16, O::VLogF16, O::VExpF16, O::BufferStoreDword, - O::SEndpgm}}; + return {"VectorSpecialF16Ops", + code, + {}, + {0x00008000u, 0xb8005678u, 0x00003800u, 0x00000000u, 0x0000fe00u, + 0x00000000u}, + {O::VMovB32, O::VRcpF16, O::VRsqF16, O::VLogF16, O::VExpF16, + O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorWritelaneIgnoresExecMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 2, 0xaaaaaaaau); - AppendSMovLiteral(&code, 4, 0x12345678u); - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); - AppendVop3(&code, 0x361, 2, 4, InlineU32(1)); - code.push_back(EncodeSMovB32(126, InlineU32(0xf))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); - code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 2, 0xaaaaaaaau); + AppendSMovLiteral(&code, 4, 0x12345678u); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + AppendVop3(&code, 0x361, 2, 4, InlineU32(1)); + code.push_back(EncodeSMovB32(126, InlineU32(0xf))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "VectorWritelaneIgnoresExecMask"; - test.code = code; - test.expected = {0xaaaaaaaau, 0x12345678u, 0xaaaaaaaau, 0xaaaaaaaau}; - test.opcodes = {O::VMovB32, O::SMovB32, O::VWritelaneB32, - O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorWritelaneIgnoresExecMask"; + test.code = code; + test.expected = {0xaaaaaaaau, 0x12345678u, 0xaaaaaaaau, 0xaaaaaaaau}; + test.opcodes = {O::VMovB32, O::SMovB32, O::VWritelaneB32, + O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorReadlaneFromInactiveWrittenLane() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 4, 0x12345678u); - AppendVop3(&code, 0x361, 2, 4, InlineU32(4)); - AppendSMovLiteral(&code, 4, 0xdeadbeefu); - AppendVop3(&code, 0x360, 5, Vgpr(2), InlineU32(4)); - AppendStoreSgpr(&code, 5, 0); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 4, 0x12345678u); + AppendVop3(&code, 0x361, 2, 4, InlineU32(4)); + AppendSMovLiteral(&code, 4, 0xdeadbeefu); + AppendVop3(&code, 0x360, 5, Vgpr(2), InlineU32(4)); + AppendStoreSgpr(&code, 5, 0); + AppendEnd(&code); - TestCase test; - test.name = "VectorReadlaneInactiveWrittenLane"; - test.code = code; - test.expected = {0x12345678u}; - test.opcodes = {O::SMovB32, O::VWritelaneB32, O::VReadlaneB32, - O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.forbidden_spirv = {"OpGroupNonUniformShuffle"}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorReadlaneInactiveWrittenLane"; + test.code = code; + test.expected = {0x12345678u}; + test.opcodes = {O::SMovB32, O::VWritelaneB32, O::VReadlaneB32, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.forbidden_spirv = {"OpGroupNonUniformShuffle"}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorLaneWave32RuntimeSelectorWraps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSop2(0x00, 5, 4, InlineU32(33))); - AppendSMovLiteral(&code, 6, 0x12345678u); - AppendVMovU32(&code, 2, 0); - AppendVop3(&code, 0x361, 2, 6, 5); - AppendVop3(&code, 0x360, 7, Vgpr(2), InlineU32(1)); - AppendStoreSgpr(&code, 7, 0); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSop2(0x00, 5, 4, InlineU32(33))); + AppendSMovLiteral(&code, 6, 0x12345678u); + AppendVMovU32(&code, 2, 0); + AppendVop3(&code, 0x361, 2, 6, 5); + AppendVop3(&code, 0x360, 7, Vgpr(2), InlineU32(1)); + AppendStoreSgpr(&code, 7, 0); + AppendEnd(&code); - TestCase test; - test.name = "VectorLaneWave32RuntimeSelectorWraps"; - test.code = code; - test.expected = {0x12345678u}; - test.opcodes = {O::SAddU32, O::SMovB32, O::VWritelaneB32, O::VReadlaneB32, - O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.required_spirv = {"OpGroupNonUniformShuffle"}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.wave_size = 32; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 4; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorLaneWave32RuntimeSelectorWraps"; + test.code = code; + test.expected = {0x12345678u}; + test.opcodes = {O::SAddU32, O::SMovB32, O::VWritelaneB32, + O::VReadlaneB32, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}; + test.required_spirv = {"OpGroupNonUniformShuffle"}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 4; + test.has_compute_info = true; + return test; } TestCase VectorPermlanex16() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 3, 0x76543210u); - AppendVMovLiteral(&code, 4, 0xfedcba98u); - code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0)); - AppendVMovLiteral(&code, 5, 0xfeedbabeu); - AppendVop3(&code, 0x378, 1, Vgpr(5), Vgpr(3), Vgpr(4)); - AppendBufferStoreDword(&code, 1, 2); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 3, 0x76543210u); + AppendVMovLiteral(&code, 4, 0xfedcba98u); + code.push_back(EncodeVop2(0x1a, 2, InlineU32(2), 0)); + AppendVMovLiteral(&code, 5, 0xfeedbabeu); + AppendVop3(&code, 0x378, 1, Vgpr(5), Vgpr(3), Vgpr(4)); + AppendBufferStoreDword(&code, 1, 2); + AppendEnd(&code); - TestCase test; - test.name = "VectorPermlanex16"; - test.code = code; - test.expected = std::vector(32, 0xfeedbabeu); - test.opcodes = {O::VMovB32, O::VPermlanex16B32, O::VLshlrevB32, O::BufferStoreDword, - O::SEndpgm}; - test.compute_info.threads_num[0] = 32; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorPermlanex16"; + test.code = code; + test.expected = std::vector(32, 0xfeedbabeu); + test.opcodes = {O::VMovB32, O::VPermlanex16B32, O::VLshlrevB32, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 32; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorPermlane16FetchInactiveZero() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), EncodeVop2(0x25, 1, InlineU32(10), 1), - EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), - EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), - }; - AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1); - AppendStoreVgpr(&code, 2, 0); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), EncodeVop2(0x25, 1, InlineU32(10), 1), + EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), + EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), + }; + AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1); + AppendStoreVgpr(&code, 2, 0); + AppendEnd(&code); - TestCase test; - test.name = "VectorPermlane16FetchInactiveZero"; - test.code = code; - test.expected = {0}; - test.opcodes = {O::VMovB32, O::VAddNcU32, O::SMovB32, - O::VPermlane16B32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorPermlane16FetchInactiveZero"; + test.code = code; + test.expected = {0}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::SMovB32, + O::VPermlane16B32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorPermlane16FetchInactiveFi() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1), - EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), - EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), - }; - AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1); - AppendStoreVgpr(&code, 2, 0); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, Vgpr(0)), EncodeVop2(0x25, 1, InlineU32(10), 1), + EncodeSMovB32(126, InlineU32(1)), EncodeSMovB32(127, InlineU32(0)), + EncodeSMovB32(0, InlineU32(1)), EncodeSMovB32(1, InlineU32(0)), + }; + AppendVop3(&code, 0x377, 2, Vgpr(1), 0, 1, 0, 1); + AppendStoreVgpr(&code, 2, 0); + AppendEnd(&code); - TestCase test; - test.name = "VectorPermlane16FetchInactiveFi"; - test.code = code; - test.expected = {11}; - test.opcodes = {O::VMovB32, O::VAddNcU32, O::SMovB32, - O::VPermlane16B32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorPermlane16FetchInactiveFi"; + test.code = code; + test.expected = {11}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::SMovB32, + O::VPermlane16B32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorDppQuadPermuteReverse() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 100); - code.push_back(EncodeVop2(0x25, 2, 250, 1)); - code.push_back(EncodeVop2Dpp(0, 0x01b)); - code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 2, 250, 1)); + code.push_back(EncodeVop2Dpp(0, 0x01b)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "VectorDppQuadPermuteReverse"; - test.code = code; - test.expected = {103, 102, 101, 100, 107, 106, 105, 104}; - test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 8; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorDppQuadPermuteReverse"; + test.code = code; + test.expected = {103, 102, 101, 100, 107, 106, 105, 104}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorDppBankMaskPreservesDestination() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 2, 0xaaaaaaaau); - AppendVMovU32(&code, 1, 100); - code.push_back(EncodeVop2(0x25, 2, 250, 1)); - code.push_back(EncodeVop2Dpp(0, 0x0e4, 0xf, 0xe)); - code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 2, 0xaaaaaaaau); + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 2, 250, 1)); + code.push_back(EncodeVop2Dpp(0, 0x0e4, 0xf, 0xe)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "VectorDppBankMaskPreservesDestination"; - test.code = code; - test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 104, 105, 106, 107}; - test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 8; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorDppBankMaskPreservesDestination"; + test.code = code; + test.expected = {0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, 0xaaaaaaaau, + 104, 105, 106, 107}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorDppBoundsControlZeroPreservesDestination() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 2, 0xaaaaaaaau); - AppendVMovU32(&code, 1, 100); - code.push_back(EncodeVop2(0x25, 2, 250, 1)); - code.push_back(EncodeVop2Dpp(0, 0x111)); - code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 2, 0xaaaaaaaau); + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeVop2(0x25, 2, 250, 1)); + code.push_back(EncodeVop2Dpp(0, 0x111)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "VectorDppBoundsControlZeroPreservesDestination"; - test.code = code; - test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106}; - test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 8; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "VectorDppBoundsControlZeroPreservesDestination"; + test.code = code; + test.expected = {0xaaaaaaaau, 100, 101, 102, 103, 104, 105, 106}; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 8; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase Vop3LdexpSourceModifier() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 9, 0xbfc00000u); - code.push_back(0xd7620107u); - code.push_back(0x00018509u); - AppendStoreVgpr(&code, 7, 0); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 9, 0xbfc00000u); + code.push_back(0xd7620107u); + code.push_back(0x00018509u); + AppendStoreVgpr(&code, 7, 0); + AppendEnd(&code); - return {"Vop3LdexpSourceModifier", - code, - {}, - {0x3ec00000u}, - {O::VMovB32, O::VLdexpF32, O::BufferStoreDword, O::SEndpgm}}; + return {"Vop3LdexpSourceModifier", + code, + {}, + {0x3ec00000u}, + {O::VMovB32, O::VLdexpF32, O::BufferStoreDword, O::SEndpgm}}; } TestCase Vop1MoveRelSource() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 12, 0x11111111u); - AppendVMovLiteral(&code, 13, 0x22222222u); - AppendVMovLiteral(&code, 14, 0x33333333u); - code.push_back(EncodeSMovB32(124, InlineU32(2))); - code.push_back(0x7e6e870cu); - AppendStoreVgpr(&code, 55, 0); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 12, 0x11111111u); + AppendVMovLiteral(&code, 13, 0x22222222u); + AppendVMovLiteral(&code, 14, 0x33333333u); + code.push_back(EncodeSMovB32(124, InlineU32(2))); + code.push_back(0x7e6e870cu); + AppendStoreVgpr(&code, 55, 0); + AppendEnd(&code); - return {"Vop1MoveRelSource", - code, - {}, - {0x33333333u}, - {O::VMovB32, O::SMovB32, O::VMovrelsB32, O::BufferStoreDword, O::SEndpgm}}; + return {"Vop1MoveRelSource", + code, + {}, + {0x33333333u}, + {O::VMovB32, O::SMovB32, O::VMovrelsB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase Vop1MoveRelDestination() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 1, 0x12345678u); - AppendVMovLiteral(&code, 5, 0xaaaaaaaau); - AppendVMovLiteral(&code, 6, 0xbbbbbbbbu); - AppendVMovLiteral(&code, 7, 0xccccccccu); - code.push_back(EncodeSMovB32(124, InlineU32(2))); - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); - code.push_back(EncodeVop1(0x42, 5, Vgpr(1))); - code.push_back(EncodeSMovB32(126, InlineU32(0xf))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); - code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); - AppendBufferStoreDword(&code, 7, 4); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 5, 0xaaaaaaaau); + AppendVMovLiteral(&code, 6, 0xbbbbbbbbu); + AppendVMovLiteral(&code, 7, 0xccccccccu); + code.push_back(EncodeSMovB32(124, InlineU32(2))); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVop1(0x42, 5, Vgpr(1))); + code.push_back(EncodeSMovB32(126, InlineU32(0xf))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + code.push_back(EncodeVop2(0x1a, 4, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 7, 4); + AppendEnd(&code); - TestCase test; - test.name = "Vop1MoveRelDestination"; - test.code = code; - test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu}; - test.opcodes = {O::VMovB32, O::SMovB32, O::VMovreldB32, - O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "Vop1MoveRelDestination"; + test.code = code; + test.expected = {0x12345678u, 0xccccccccu, 0xccccccccu, 0xccccccccu}; + test.opcodes = {O::VMovB32, O::SMovB32, O::VMovreldB32, + O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase VectorFloatSpecialOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x40000000u); - AppendVMovLiteral(&code, 2, 0x40400000u); - AppendVMovLiteral(&code, 3, 0x40800000u); - AppendVMovLiteral(&code, 4, 0x40003c00u); - AppendVMovLiteral(&code, 5, 0x44004200u); - AppendVMovLiteral(&code, 6, 0x3f800000u); - AppendVMovLiteral(&code, 7, 0x3f800000u); - AppendVMovLiteral(&code, 8, 0xbf800000u); - AppendVMovLiteral(&code, 10, 0x3f000000u); - AppendVMovLiteral(&code, 12, 0xaaaabbbbu); - AppendVMovLiteral(&code, 13, 0xaaaabbbbu); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40400000u); + AppendVMovLiteral(&code, 3, 0x40800000u); + AppendVMovLiteral(&code, 4, 0x40003c00u); + AppendVMovLiteral(&code, 5, 0x44004200u); + AppendVMovLiteral(&code, 6, 0x3f800000u); + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendVMovLiteral(&code, 8, 0xbf800000u); + AppendVMovLiteral(&code, 10, 0x3f000000u); + AppendVMovLiteral(&code, 12, 0xaaaabbbbu); + AppendVMovLiteral(&code, 13, 0xaaaabbbbu); - code.push_back(EncodeVop2(0x02, 6, Vgpr(4), 5)); - AppendVop3(&code, 0x368, 9, Vgpr(7), Vgpr(8)); - AppendVop3(&code, 0x369, 11, Vgpr(7), Vgpr(10)); - AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(2), Vgpr(0)); - AppendVop3p(&code, 0x22, 13, Vgpr(1), Vgpr(2), Vgpr(0)); - AppendVop3(&code, 0x144, 14, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x145, 15, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x146, 16, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x147, 17, Vgpr(0), Vgpr(1), Vgpr(2)); + code.push_back(EncodeVop2(0x02, 6, Vgpr(4), 5)); + AppendVop3(&code, 0x368, 9, Vgpr(7), Vgpr(8)); + AppendVop3(&code, 0x369, 11, Vgpr(7), Vgpr(10)); + AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(2), Vgpr(0)); + AppendVop3p(&code, 0x22, 13, Vgpr(1), Vgpr(2), Vgpr(0)); + AppendVop3(&code, 0x144, 14, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x145, 15, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x146, 16, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x147, 17, Vgpr(0), Vgpr(1), Vgpr(2)); - const u32 results[] = {6, 9, 11, 12, 13, 14, 15, 16, 17}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {6, 9, 11, 12, 13, 14, 15, 16, 17}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"VectorFloatSpecialOps", - code, - {}, - {0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu, 0x40800000u, - 0x3f800000u, 0xc0000000u, 0x40c00000u}, - {O::VMovB32, O::VDot2cF32F16, O::VCvtPknormI16F32, O::VCvtPknormU16F32, O::VMadMixloF16, - O::VMadMixhiF16, O::VCubeidF32, O::VCubescF32, O::VCubetcF32, O::VCubemaF32, - O::BufferStoreDword, O::SEndpgm}}; + return {"VectorFloatSpecialOps", + code, + {}, + {0x41400000u, 0x80017fffu, 0x8000ffffu, 0xaaaa4700u, 0x4700bbbbu, + 0x40800000u, 0x3f800000u, 0xc0000000u, 0x40c00000u}, + {O::VMovB32, O::VDot2cF32F16, O::VCvtPknormI16F32, + O::VCvtPknormU16F32, O::VMadMixloF16, O::VMadMixhiF16, O::VCubeidF32, + O::VCubescF32, O::VCubetcF32, O::VCubemaF32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase MadMixF16LiteralHalfSourceUsesOpsel() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 12, 0xaaaabbbbu); - AppendVMovLiteral(&code, 13, 0x11112222u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 12, 0xaaaabbbbu); + AppendVMovLiteral(&code, 13, 0x11112222u); - AppendVop3p(&code, 0x21, 12, 255u, Vgpr(0), Vgpr(0), 0x1, 0x1); - code.push_back(0x40003c00u); - AppendVop3p(&code, 0x22, 13, 255u, Vgpr(0), Vgpr(0), 0x1, 0x0); - code.push_back(0x40003c00u); + AppendVop3p(&code, 0x21, 12, 255u, Vgpr(0), Vgpr(0), 0x1, 0x1); + code.push_back(0x40003c00u); + AppendVop3p(&code, 0x22, 13, 255u, Vgpr(0), Vgpr(0), 0x1, 0x0); + code.push_back(0x40003c00u); - AppendStoreVgpr(&code, 12, 0); - AppendStoreVgpr(&code, 13, 1); - AppendEnd(&code); + AppendStoreVgpr(&code, 12, 0); + AppendStoreVgpr(&code, 13, 1); + AppendEnd(&code); - return {"MadMixF16LiteralHalfSourceUsesOpsel", - code, - {}, - {0xaaaa4200u, 0x40002222u}, - {O::VMovB32, O::VMadMixloF16, O::VMadMixhiF16, O::BufferStoreDword, O::SEndpgm}}; + return {"MadMixF16LiteralHalfSourceUsesOpsel", + code, + {}, + {0xaaaa4200u, 0x40002222u}, + {O::VMovB32, O::VMadMixloF16, O::VMadMixhiF16, O::BufferStoreDword, + O::SEndpgm}}; } TestCase MadMixF16NegHiIsAbsAndNegIsIndependent() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0xc0000000u); - AppendVMovLiteral(&code, 2, 0x40000000u); - AppendVMovLiteral(&code, 12, 0xaaaabbbbu); - AppendVMovLiteral(&code, 13, 0x11112222u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0xc0000000u); + AppendVMovLiteral(&code, 2, 0x40000000u); + AppendVMovLiteral(&code, 12, 0xaaaabbbbu); + AppendVMovLiteral(&code, 13, 0x11112222u); - AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(0), Vgpr(0), 0, 0, 0x1); - AppendVop3p(&code, 0x22, 13, Vgpr(2), Vgpr(0), Vgpr(0), 0, 0, 0, 0x1); + AppendVop3p(&code, 0x21, 12, Vgpr(1), Vgpr(0), Vgpr(0), 0, 0, 0x1); + AppendVop3p(&code, 0x22, 13, Vgpr(2), Vgpr(0), Vgpr(0), 0, 0, 0, 0x1); - AppendStoreVgpr(&code, 12, 0); - AppendStoreVgpr(&code, 13, 1); - AppendEnd(&code); + AppendStoreVgpr(&code, 12, 0); + AppendStoreVgpr(&code, 13, 1); + AppendEnd(&code); - return {"MadMixF16NegHiIsAbsAndNegIsIndependent", - code, - {}, - {0xaaaa4200u, 0xbc002222u}, - {O::VMovB32, O::VMadMixloF16, O::VMadMixhiF16, O::BufferStoreDword, O::SEndpgm}}; + return {"MadMixF16NegHiIsAbsAndNegIsIndependent", + code, + {}, + {0xaaaa4200u, 0xbc002222u}, + {O::VMovB32, O::VMadMixloF16, O::VMadMixhiF16, O::BufferStoreDword, + O::SEndpgm}}; } TestCase VectorVop3FmaF16UsesRdna2Opcode34b() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 1, 0x40003c00u); - AppendVMovLiteral(&code, 2, 0x44004200u); - AppendVMovLiteral(&code, 3, 0x3c003c00u); - AppendVMovLiteral(&code, 10, 0xaaaa5555u); - AppendVMovLiteral(&code, 11, 0xbbbb5555u); + std::vector code; + AppendVMovLiteral(&code, 1, 0x40003c00u); + AppendVMovLiteral(&code, 2, 0x44004200u); + AppendVMovLiteral(&code, 3, 0x3c003c00u); + AppendVMovLiteral(&code, 10, 0xaaaa5555u); + AppendVMovLiteral(&code, 11, 0xbbbb5555u); - AppendVop3(&code, 0x34bu, 10, Vgpr(1), Vgpr(2), Vgpr(3)); - AppendVop3(&code, 0x34bu, 11, Vgpr(1), Vgpr(2), Vgpr(3), 0, 0xfu); + AppendVop3(&code, 0x34bu, 10, Vgpr(1), Vgpr(2), Vgpr(3)); + AppendVop3(&code, 0x34bu, 11, Vgpr(1), Vgpr(2), Vgpr(3), 0, 0xfu); - AppendStoreVgpr(&code, 10, 0); - AppendStoreVgpr(&code, 11, 1); - AppendEnd(&code); + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendEnd(&code); - return {"VectorVop3FmaF16UsesRdna2Opcode34b", - code, - {}, - {0xaaaa4400u, 0x48805555u}, - {O::VMovB32, O::VFmaF16, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorVop3FmaF16UsesRdna2Opcode34b", + code, + {}, + {0xaaaa4400u, 0x48805555u}, + {O::VMovB32, O::VFmaF16, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorFloatArithmeticOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x40000000u); - AppendVMovLiteral(&code, 2, 0x40800000u); - AppendVMovLiteral(&code, 3, 0x40a00000u); - AppendVMovLiteral(&code, 4, 0x40400000u); - AppendVMovLiteral(&code, 11, 0x40800000u); - AppendVMovLiteral(&code, 17, 0x40800000u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40800000u); + AppendVMovLiteral(&code, 3, 0x40a00000u); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 11, 0x40800000u); + AppendVMovLiteral(&code, 17, 0x40800000u); - code.push_back(EncodeVop2(0x03, 5, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x04, 6, Vgpr(3), 1)); - code.push_back(EncodeVop2(0x05, 7, Vgpr(1), 3)); - code.push_back(EncodeVop2(0x08, 8, Vgpr(1), 2)); - code.push_back(EncodeVop1(0x01, 21, Vgpr(8))); - code.push_back(EncodeVop2(0x0f, 9, Vgpr(3), 1)); - code.push_back(EncodeVop2(0x10, 10, Vgpr(3), 1)); - code.push_back(EncodeVop2(0x1f, 11, Vgpr(1), 4)); - AppendVop3(&code, 0x141, 12, Vgpr(1), Vgpr(4), Vgpr(2)); - AppendVop3(&code, 0x14b, 13, Vgpr(1), Vgpr(4), Vgpr(2)); - AppendVop3(&code, 0x151, 14, Vgpr(3), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x154, 15, Vgpr(3), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x157, 16, Vgpr(3), Vgpr(1), Vgpr(2)); - code.push_back(EncodeVop2(0x20, 18, Vgpr(1), 17)); - code.push_back(0x3f800000u); - code.push_back(EncodeVop2(0x21, 19, Vgpr(1), 4)); - code.push_back(0x40000000u); - code.push_back(EncodeVop2(0x03, 20, 249, 1)); - code.push_back(EncodeVop2Sdwa(0)); - AppendSMovLiteral(&code, 70, 0x40000000u); - code.push_back(0x06088cf9u); - code.push_back(0x868606f2u); - AppendSMovLiteral(&code, 26, 0x40000000u); - code.push_back(0x081034f9u); - code.push_back(0x868606f2u); - code.push_back(0xd51f8011u); - code.push_back(0x00020d0bu); - AppendVMovLiteral(&code, 22, 0xbf800000u); - code.push_back(EncodeVop2(0x05, 23, 249, 26)); - code.push_back(EncodeVop2Sdwa(22, 6, 0, 6, 6, 0, 0, 0, 1, 0, 0, 0, 1)); - AppendVMovLiteral(&code, 24, 0x3f800000u); - AppendVMovLiteral(&code, 25, 0x3f800000u); - code.push_back(EncodeVop2(0x03, 26, 249, 25)); - code.push_back(EncodeVop2Sdwa(24, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)); - AppendVMovLiteral(&code, 27, 0xbf800000u); - AppendVMovLiteral(&code, 28, 0x40000000u); - AppendVop3(&code, 0x104, 29, Vgpr(27), Vgpr(28), 0, 1); - AppendVop3(&code, 0x103, 30, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1); - AppendVop3(&code, 0x151, 32, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0, false, 1); - AppendVMovLiteral(&code, 33, 0xbf800000u); - AppendVMovLiteral(&code, 34, 0x40000000u); - code.push_back(EncodeVop2(0x03, 35, 250, 34)); - code.push_back(EncodeVop2Dpp(33, 0, 0xf, 0xf, 0, 1)); + code.push_back(EncodeVop2(0x03, 5, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x04, 6, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x05, 7, Vgpr(1), 3)); + code.push_back(EncodeVop2(0x08, 8, Vgpr(1), 2)); + code.push_back(EncodeVop1(0x01, 21, Vgpr(8))); + code.push_back(EncodeVop2(0x0f, 9, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x10, 10, Vgpr(3), 1)); + code.push_back(EncodeVop2(0x1f, 11, Vgpr(1), 4)); + AppendVop3(&code, 0x141, 12, Vgpr(1), Vgpr(4), Vgpr(2)); + AppendVop3(&code, 0x14b, 13, Vgpr(1), Vgpr(4), Vgpr(2)); + AppendVop3(&code, 0x151, 14, Vgpr(3), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x154, 15, Vgpr(3), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x157, 16, Vgpr(3), Vgpr(1), Vgpr(2)); + code.push_back(EncodeVop2(0x20, 18, Vgpr(1), 17)); + code.push_back(0x3f800000u); + code.push_back(EncodeVop2(0x21, 19, Vgpr(1), 4)); + code.push_back(0x40000000u); + code.push_back(EncodeVop2(0x03, 20, 249, 1)); + code.push_back(EncodeVop2Sdwa(0)); + AppendSMovLiteral(&code, 70, 0x40000000u); + code.push_back(0x06088cf9u); + code.push_back(0x868606f2u); + AppendSMovLiteral(&code, 26, 0x40000000u); + code.push_back(0x081034f9u); + code.push_back(0x868606f2u); + code.push_back(0xd51f8011u); + code.push_back(0x00020d0bu); + AppendVMovLiteral(&code, 22, 0xbf800000u); + code.push_back(EncodeVop2(0x05, 23, 249, 26)); + code.push_back(EncodeVop2Sdwa(22, 6, 0, 6, 6, 0, 0, 0, 1, 0, 0, 0, 1)); + AppendVMovLiteral(&code, 24, 0x3f800000u); + AppendVMovLiteral(&code, 25, 0x3f800000u); + code.push_back(EncodeVop2(0x03, 26, 249, 25)); + code.push_back(EncodeVop2Sdwa(24, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)); + AppendVMovLiteral(&code, 27, 0xbf800000u); + AppendVMovLiteral(&code, 28, 0x40000000u); + AppendVop3(&code, 0x104, 29, Vgpr(27), Vgpr(28), 0, 1); + AppendVop3(&code, 0x103, 30, Vgpr(0), Vgpr(1), 0, 0, 0, false, 1); + AppendVop3(&code, 0x151, 32, Vgpr(0), Vgpr(1), Vgpr(2), 0, 0, false, 1); + AppendVMovLiteral(&code, 33, 0xbf800000u); + AppendVMovLiteral(&code, 34, 0x40000000u); + code.push_back(EncodeVop2(0x03, 35, 250, 34)); + code.push_back(EncodeVop2Dpp(33, 0, 0xf, 0xf, 0, 1)); - const u32 results[] = {5, 6, 7, 21, 9, 10, 11, 12, 13, 14, 15, 16, - 18, 19, 20, 4, 8, 17, 23, 26, 29, 30, 32, 35}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {5, 6, 7, 21, 9, 10, 11, 12, 13, 14, 15, 16, + 18, 19, 20, 4, 8, 17, 23, 26, 29, 30, 32, 35}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"VectorFloatArithmeticOps", - code, - {}, - {0x40400000u, 0x40400000u, 0x40400000u, 0x41000000u, 0x40000000u, 0x40a00000u, - 0x41200000u, 0x41200000u, 0x41200000u, 0x40000000u, 0x40a00000u, 0x40800000u, - 0x40c00000u, 0x41000000u, 0x40400000u, 0x40400000u, 0xbf800000u, 0x3f800000u, - 0x3f800000u, 0x40800000u, 0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u}, - {O::SMovB32, O::VMovB32, O::VAddF32, O::VSubF32, O::VSubrevF32, O::VMulF32, O::VMinF32, - O::VMaxF32, O::VMacF32, O::VMadF32, O::VFmaF32, O::VMin3F32, O::VMax3F32, O::VMed3F32, - O::VMadmkF32, O::VMadakF32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorFloatArithmeticOps", + code, + {}, + {0x40400000u, 0x40400000u, 0x40400000u, 0x41000000u, 0x40000000u, + 0x40a00000u, 0x41200000u, 0x41200000u, 0x41200000u, 0x40000000u, + 0x40a00000u, 0x40800000u, 0x40c00000u, 0x41000000u, 0x40400000u, + 0x40400000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, 0x40800000u, + 0xbf800000u, 0x40c00000u, 0x40000000u, 0x40400000u}, + {O::SMovB32, O::VMovB32, O::VAddF32, O::VSubF32, O::VSubrevF32, + O::VMulF32, O::VMinF32, O::VMaxF32, O::VMacF32, O::VMadF32, + O::VFmaF32, O::VMin3F32, O::VMax3F32, O::VMed3F32, O::VMadmkF32, + O::VMadakF32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorMinMaxF32NanAndSignedZeroEdges() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x40000000u); - AppendVMovLiteral(&code, 1, 0x7fc00000u); - AppendVMovLiteral(&code, 2, 0x00000000u); - AppendVMovLiteral(&code, 3, 0x80000000u); - AppendVMovLiteral(&code, 4, 0x40400000u); - AppendVMovLiteral(&code, 5, 0x3f800000u); - AppendVMovLiteral(&code, 6, 0x7fa00001u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x40000000u); + AppendVMovLiteral(&code, 1, 0x7fc00000u); + AppendVMovLiteral(&code, 2, 0x00000000u); + AppendVMovLiteral(&code, 3, 0x80000000u); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 5, 0x3f800000u); + AppendVMovLiteral(&code, 6, 0x7fa00001u); - code.push_back(EncodeVop2(0x0f, 10, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x10, 11, Vgpr(0), 1)); - code.push_back(EncodeVop2(0x0f, 12, Vgpr(3), 2)); - code.push_back(EncodeVop2(0x10, 13, Vgpr(2), 3)); - code.push_back(EncodeVop2(0x0f, 14, Vgpr(6), 0)); - code.push_back(EncodeVop2(0x10, 15, Vgpr(0), 6)); - AppendVop3(&code, 0x151, 16, Vgpr(0), Vgpr(1), Vgpr(4)); - AppendVop3(&code, 0x154, 17, Vgpr(0), Vgpr(1), Vgpr(5)); + code.push_back(EncodeVop2(0x0f, 10, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x10, 11, Vgpr(0), 1)); + code.push_back(EncodeVop2(0x0f, 12, Vgpr(3), 2)); + code.push_back(EncodeVop2(0x10, 13, Vgpr(2), 3)); + code.push_back(EncodeVop2(0x0f, 14, Vgpr(6), 0)); + code.push_back(EncodeVop2(0x10, 15, Vgpr(0), 6)); + AppendVop3(&code, 0x151, 16, Vgpr(0), Vgpr(1), Vgpr(4)); + AppendVop3(&code, 0x154, 17, Vgpr(0), Vgpr(1), Vgpr(5)); - for (u32 i = 0; i < 8; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 8; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"VectorMinMaxF32NanAndSignedZeroEdges", - code, - {}, - {0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x7fe00001u, 0x7fe00001u, - 0x40000000u, 0x40000000u}, - {O::VMovB32, O::VMinF32, O::VMaxF32, O::VMin3F32, O::VMax3F32, O::BufferStoreDword, - O::SEndpgm}}; + return {"VectorMinMaxF32NanAndSignedZeroEdges", + code, + {}, + {0x40000000u, 0x40000000u, 0x80000000u, 0x00000000u, 0x7fe00001u, + 0x7fe00001u, 0x40000000u, 0x40000000u}, + {O::VMovB32, O::VMinF32, O::VMaxF32, O::VMin3F32, O::VMax3F32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorMed3F32NanUsesMin3Path() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x40000000u); - AppendVMovLiteral(&code, 1, 0x40400000u); - AppendVMovLiteral(&code, 2, 0x7fa00001u); - AppendVMovLiteral(&code, 3, 0x7fc00000u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x40000000u); + AppendVMovLiteral(&code, 1, 0x40400000u); + AppendVMovLiteral(&code, 2, 0x7fa00001u); + AppendVMovLiteral(&code, 3, 0x7fc00000u); - AppendVop3(&code, 0x157, 10, Vgpr(0), Vgpr(1), Vgpr(2)); - AppendVop3(&code, 0x157, 11, Vgpr(0), Vgpr(1), Vgpr(3)); - AppendVop3(&code, 0x157, 12, Vgpr(2), Vgpr(0), Vgpr(1)); + AppendVop3(&code, 0x157, 10, Vgpr(0), Vgpr(1), Vgpr(2)); + AppendVop3(&code, 0x157, 11, Vgpr(0), Vgpr(1), Vgpr(3)); + AppendVop3(&code, 0x157, 12, Vgpr(2), Vgpr(0), Vgpr(1)); - AppendStoreVgpr(&code, 10, 0); - AppendStoreVgpr(&code, 11, 1); - AppendStoreVgpr(&code, 12, 2); - AppendEnd(&code); + AppendStoreVgpr(&code, 10, 0); + AppendStoreVgpr(&code, 11, 1); + AppendStoreVgpr(&code, 12, 2); + AppendEnd(&code); - return {"VectorMed3F32NanUsesMin3Path", - code, - {}, - {0x7fe00001u, 0x40000000u, 0x40400000u}, - {O::VMovB32, O::VMed3F32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorMed3F32NanUsesMin3Path", + code, + {}, + {0x7fe00001u, 0x40000000u, 0x40400000u}, + {O::VMovB32, O::VMed3F32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorFloatConversionOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xfffffffdu); - code.push_back(EncodeVop1(0x01, 1, InlineU32(7))); - AppendVMovLiteral(&code, 2, 0x40e00000u); - AppendVMovLiteral(&code, 3, 0xc0400000u); - AppendVMovLiteral(&code, 4, 0x3f800000u); - AppendVMovLiteral(&code, 5, 0x00003c00u); - AppendVMovLiteral(&code, 6, 0x40700000u); - AppendVMovLiteral(&code, 7, 0x44332211u); - AppendVMovLiteral(&code, 8, 0x40000000u); - AppendVMovLiteral(&code, 9, 0x40800000u); - AppendVMovLiteral(&code, 10, 0x40100000u); - AppendVMovLiteral(&code, 11, 0x40300000u); - code.push_back(EncodeVop1(0x01, 12, InlineU32(0))); - code.push_back(EncodeVop1(0x01, 37, InlineU32(15))); + std::vector code; + AppendVMovLiteral(&code, 0, 0xfffffffdu); + code.push_back(EncodeVop1(0x01, 1, InlineU32(7))); + AppendVMovLiteral(&code, 2, 0x40e00000u); + AppendVMovLiteral(&code, 3, 0xc0400000u); + AppendVMovLiteral(&code, 4, 0x3f800000u); + AppendVMovLiteral(&code, 5, 0x00003c00u); + AppendVMovLiteral(&code, 6, 0x40700000u); + AppendVMovLiteral(&code, 7, 0x44332211u); + AppendVMovLiteral(&code, 8, 0x40000000u); + AppendVMovLiteral(&code, 9, 0x40800000u); + AppendVMovLiteral(&code, 10, 0x40100000u); + AppendVMovLiteral(&code, 11, 0x40300000u); + code.push_back(EncodeVop1(0x01, 12, InlineU32(0))); + code.push_back(EncodeVop1(0x01, 37, InlineU32(15))); - code.push_back(EncodeVop1(0x05, 13, Vgpr(0))); - code.push_back(EncodeVop1(0x06, 14, Vgpr(1))); - code.push_back(EncodeVop1(0x07, 15, Vgpr(2))); - code.push_back(EncodeVop1(0x08, 16, Vgpr(3))); - code.push_back(EncodeVop1(0x0a, 17, Vgpr(4))); - code.push_back(EncodeVop1(0x0b, 18, Vgpr(5))); - code.push_back(EncodeVop1(0x0c, 42, Vgpr(11))); - code.push_back(EncodeVop1(0x0d, 19, Vgpr(6))); - code.push_back(EncodeVop1(0x11, 20, Vgpr(7))); - code.push_back(EncodeVop1(0x12, 21, Vgpr(7))); - code.push_back(EncodeVop1(0x13, 22, Vgpr(7))); - code.push_back(EncodeVop1(0x14, 23, Vgpr(7))); - code.push_back(EncodeVop1(0x2a, 24, Vgpr(8))); - code.push_back(EncodeVop1(0x20, 25, Vgpr(10))); - code.push_back(EncodeVop1(0x21, 26, Vgpr(11))); - code.push_back(EncodeVop1(0x22, 27, Vgpr(10))); - code.push_back(EncodeVop1(0x23, 28, Vgpr(10))); - code.push_back(EncodeVop1(0x24, 29, Vgpr(11))); - code.push_back(EncodeVop1(0x25, 30, Vgpr(4))); - code.push_back(EncodeVop1(0x27, 36, Vgpr(9))); - code.push_back(EncodeVop1(0x2e, 32, Vgpr(9))); - code.push_back(EncodeVop1(0x33, 33, Vgpr(9))); - code.push_back(EncodeVop1(0x35, 34, Vgpr(12))); - code.push_back(EncodeVop1(0x36, 35, Vgpr(12))); - code.push_back(EncodeVop1(0x0e, 38, Vgpr(37))); - code.push_back(EncodeVop1(0x2a, 39, 249)); - code.push_back(EncodeVop1Sdwa(8, 6, 0, 6, 0, 0, 0, 0, 0, 1)); - AppendVMovLiteral(&code, 40, 0xc0000000u); - AppendVop3(&code, 0x1aa, 41, Vgpr(40), 0, 0, 1, 0, false, 1); - AppendVMovLiteral(&code, 2, 0xc0003c00u); // low=1.0h, high=-2.0h - code.push_back(0x7e1016f9u); - code.push_back(0x00250602u); + code.push_back(EncodeVop1(0x05, 13, Vgpr(0))); + code.push_back(EncodeVop1(0x06, 14, Vgpr(1))); + code.push_back(EncodeVop1(0x07, 15, Vgpr(2))); + code.push_back(EncodeVop1(0x08, 16, Vgpr(3))); + code.push_back(EncodeVop1(0x0a, 17, Vgpr(4))); + code.push_back(EncodeVop1(0x0b, 18, Vgpr(5))); + code.push_back(EncodeVop1(0x0c, 42, Vgpr(11))); + code.push_back(EncodeVop1(0x0d, 19, Vgpr(6))); + code.push_back(EncodeVop1(0x11, 20, Vgpr(7))); + code.push_back(EncodeVop1(0x12, 21, Vgpr(7))); + code.push_back(EncodeVop1(0x13, 22, Vgpr(7))); + code.push_back(EncodeVop1(0x14, 23, Vgpr(7))); + code.push_back(EncodeVop1(0x2a, 24, Vgpr(8))); + code.push_back(EncodeVop1(0x20, 25, Vgpr(10))); + code.push_back(EncodeVop1(0x21, 26, Vgpr(11))); + code.push_back(EncodeVop1(0x22, 27, Vgpr(10))); + code.push_back(EncodeVop1(0x23, 28, Vgpr(10))); + code.push_back(EncodeVop1(0x24, 29, Vgpr(11))); + code.push_back(EncodeVop1(0x25, 30, Vgpr(4))); + code.push_back(EncodeVop1(0x27, 36, Vgpr(9))); + code.push_back(EncodeVop1(0x2e, 32, Vgpr(9))); + code.push_back(EncodeVop1(0x33, 33, Vgpr(9))); + code.push_back(EncodeVop1(0x35, 34, Vgpr(12))); + code.push_back(EncodeVop1(0x36, 35, Vgpr(12))); + code.push_back(EncodeVop1(0x0e, 38, Vgpr(37))); + code.push_back(EncodeVop1(0x2a, 39, 249)); + code.push_back(EncodeVop1Sdwa(8, 6, 0, 6, 0, 0, 0, 0, 0, 1)); + AppendVMovLiteral(&code, 40, 0xc0000000u); + AppendVop3(&code, 0x1aa, 41, Vgpr(40), 0, 0, 1, 0, false, 1); + AppendVMovLiteral(&code, 2, 0xc0003c00u); // low=1.0h, high=-2.0h + code.push_back(0x7e1016f9u); + code.push_back(0x00250602u); - const u32 results[] = {13, 14, 15, 16, 17, 18, 42, 19, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 36, 32, 33, 34, 35, 38, 39, 41, 8}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {13, 14, 15, 16, 17, 18, 42, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 36, 32, 33, 34, 35, 38, 39, 41, 8}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return { - "VectorFloatConversionOps", - code, - {}, - {0xc0400000u, 0x40e00000u, 7, 0xfffffffdu, 0x00003c00u, 0x3f800000u, 3, - 3, 0x41880000u, 0x42080000u, 0x424c0000u, 0x42880000u, 0x3f000000u, 0x3e800000u, - 0x40000000u, 0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x3f000000u, - 0x40000000u, 0, 0x3f800000u, 0xbd800000u, 0x3f800000u, 0x3f800000u, 0x40000000u}, - {O::VMovB32, O::VCvtF32I32, O::VCvtF32U32, O::VCvtU32F32, O::VCvtI32F32, - O::VCvtF16F32, O::VCvtF32F16, O::VCvtRpiI32F32, O::VCvtFlrI32F32, O::VCvtOffF32I4, - O::VCvtF32Ubyte0, O::VCvtF32Ubyte1, O::VCvtF32Ubyte2, O::VCvtF32Ubyte3, O::VRcpF32, - O::VFractF32, O::VTruncF32, O::VCeilF32, O::VRndneF32, O::VFloorF32, - O::VExpF32, O::VLogF32, O::VRsqF32, O::VSqrtF32, O::VSinF32, - O::VCosF32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorFloatConversionOps", + code, + {}, + {0xc0400000u, 0x40e00000u, 7, 0xfffffffdu, 0x00003c00u, + 0x3f800000u, 3, 3, 0x41880000u, 0x42080000u, + 0x424c0000u, 0x42880000u, 0x3f000000u, 0x3e800000u, 0x40000000u, + 0x40400000u, 0x40000000u, 0x40000000u, 0x40000000u, 0x40000000u, + 0x3f000000u, 0x40000000u, 0, 0x3f800000u, 0xbd800000u, + 0x3f800000u, 0x3f800000u, 0x40000000u}, + {O::VMovB32, O::VCvtF32I32, O::VCvtF32U32, + O::VCvtU32F32, O::VCvtI32F32, O::VCvtF16F32, + O::VCvtF32F16, O::VCvtRpiI32F32, O::VCvtFlrI32F32, + O::VCvtOffF32I4, O::VCvtF32Ubyte0, O::VCvtF32Ubyte1, + O::VCvtF32Ubyte2, O::VCvtF32Ubyte3, O::VRcpF32, + O::VFractF32, O::VTruncF32, O::VCeilF32, + O::VRndneF32, O::VFloorF32, O::VExpF32, + O::VLogF32, O::VRsqF32, O::VSqrtF32, + O::VSinF32, O::VCosF32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase CvtF32ToIntSaturatesNaNAndOutOfRange() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x7fc00000u); - AppendVMovLiteral(&code, 1, 0xbf800000u); - AppendVMovLiteral(&code, 2, 0x7f800000u); - AppendVMovLiteral(&code, 3, 0x4f800000u); - AppendVMovLiteral(&code, 4, 0xff800000u); - AppendVMovLiteral(&code, 5, 0x4f000000u); + std::vector code; + AppendVMovLiteral(&code, 0, 0x7fc00000u); + AppendVMovLiteral(&code, 1, 0xbf800000u); + AppendVMovLiteral(&code, 2, 0x7f800000u); + AppendVMovLiteral(&code, 3, 0x4f800000u); + AppendVMovLiteral(&code, 4, 0xff800000u); + AppendVMovLiteral(&code, 5, 0x4f000000u); - code.push_back(EncodeVop1(0x07, 10, Vgpr(0))); - code.push_back(EncodeVop1(0x07, 11, Vgpr(1))); - code.push_back(EncodeVop1(0x07, 12, Vgpr(2))); - code.push_back(EncodeVop1(0x07, 13, Vgpr(3))); - code.push_back(EncodeVop1(0x08, 14, Vgpr(0))); - code.push_back(EncodeVop1(0x08, 15, Vgpr(2))); - code.push_back(EncodeVop1(0x08, 16, Vgpr(4))); - code.push_back(EncodeVop1(0x08, 17, Vgpr(5))); + code.push_back(EncodeVop1(0x07, 10, Vgpr(0))); + code.push_back(EncodeVop1(0x07, 11, Vgpr(1))); + code.push_back(EncodeVop1(0x07, 12, Vgpr(2))); + code.push_back(EncodeVop1(0x07, 13, Vgpr(3))); + code.push_back(EncodeVop1(0x08, 14, Vgpr(0))); + code.push_back(EncodeVop1(0x08, 15, Vgpr(2))); + code.push_back(EncodeVop1(0x08, 16, Vgpr(4))); + code.push_back(EncodeVop1(0x08, 17, Vgpr(5))); - for (u32 i = 0; i < 8; i++) { - AppendStoreVgpr(&code, 10 + i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 8; i++) { + AppendStoreVgpr(&code, 10 + i, i); + } + AppendEnd(&code); - return {"CvtF32ToIntSaturatesNaNAndOutOfRange", - code, - {}, - {0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u, 0x7fffffffu}, - {O::VMovB32, O::VCvtU32F32, O::VCvtI32F32, O::BufferStoreDword, O::SEndpgm}}; + return {"CvtF32ToIntSaturatesNaNAndOutOfRange", + code, + {}, + {0, 0, 0xffffffffu, 0xffffffffu, 0, 0x7fffffffu, 0x80000000u, + 0x7fffffffu}, + {O::VMovB32, O::VCvtU32F32, O::VCvtI32F32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase VectorSpecialF32FlushesDenormalInputs() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x00000001u); - code.push_back(EncodeVop1(0x27, 1, Vgpr(0))); - code.push_back(EncodeVop1(0x2a, 2, Vgpr(0))); - code.push_back(EncodeVop1(0x2e, 3, Vgpr(0))); - code.push_back(EncodeVop1(0x33, 4, Vgpr(0))); - AppendStoreVgpr(&code, 1, 0); - AppendStoreVgpr(&code, 2, 1); - AppendStoreVgpr(&code, 3, 2); - AppendStoreVgpr(&code, 4, 3); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x00000001u); + code.push_back(EncodeVop1(0x27, 1, Vgpr(0))); + code.push_back(EncodeVop1(0x2a, 2, Vgpr(0))); + code.push_back(EncodeVop1(0x2e, 3, Vgpr(0))); + code.push_back(EncodeVop1(0x33, 4, Vgpr(0))); + AppendStoreVgpr(&code, 1, 0); + AppendStoreVgpr(&code, 2, 1); + AppendStoreVgpr(&code, 3, 2); + AppendStoreVgpr(&code, 4, 3); + AppendEnd(&code); - return {"VectorSpecialF32FlushesDenormalInputs", - code, - {}, - {0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u}, - {O::VMovB32, O::VLogF32, O::VRcpF32, O::VRsqF32, O::VSqrtF32, O::BufferStoreDword, - O::SEndpgm}}; + return {"VectorSpecialF32FlushesDenormalInputs", + code, + {}, + {0xff800000u, 0x7f800000u, 0x7f800000u, 0x00000000u}, + {O::VMovB32, O::VLogF32, O::VRcpF32, O::VRsqF32, O::VSqrtF32, + O::BufferStoreDword, O::SEndpgm}}; +} + +TestCase VectorRcpIflagF32IntegerReciprocal() { + using O = ShaderOpcode; + + std::vector code; + AppendVMovU32(&code, 0, 0); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 2); + AppendVMovLiteral(&code, 3, 0xffffffffu); + for (u32 i = 0; i < 4; i++) { + code.push_back(EncodeVop1(0x06, 4 + i, Vgpr(i))); + code.push_back(EncodeVop1(0x2b, 8 + i, Vgpr(4 + i))); + } + AppendVMovLiteral(&code, 12, 0xc0000000u); + AppendVop3(&code, 0x1ab, 13, Vgpr(12), 0, 0, 1, 0, false, 1); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 8 + i, i); + } + AppendStoreVgpr(&code, 13, 4); + AppendEnd(&code); + + TestCase test; + test.name = "VectorRcpIflagF32IntegerReciprocal"; + test.code = std::move(code); + test.expected = {0x7f800000u, 0x3f800000u, 0x3f000000u, 0x2f800000u, + 0x3f800000u}; + test.opcodes = {O::VMovB32, O::VCvtF32U32, O::VRcpIflagF32, + O::BufferStoreDword, O::SEndpgm}; + test.decoded_counts = {{"v_rcp_iflag_f32", 5}}; + test.ir_counts = {{"FPRecipIFlag32", 5}}; + test.required_spirv = {"OpConvertUToF", "OpFDiv"}; + test.forbidden_spirv = {"8388608"}; + return test; } TestCase VectorSinCosMaxFiniteSpecialCases() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xff7fffffu); - code.push_back(EncodeVop1(0x35, 1, Vgpr(0))); - code.push_back(EncodeVop1(0x36, 2, Vgpr(0))); - AppendStoreVgpr(&code, 1, 0); - AppendStoreVgpr(&code, 2, 1); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0xff7fffffu); + code.push_back(EncodeVop1(0x35, 1, Vgpr(0))); + code.push_back(EncodeVop1(0x36, 2, Vgpr(0))); + AppendStoreVgpr(&code, 1, 0); + AppendStoreVgpr(&code, 2, 1); + AppendEnd(&code); - return {"VectorSinCosMaxFiniteSpecialCases", - code, - {}, - {0x00000000u, 0x3f800000u}, - {O::VMovB32, O::VSinF32, O::VCosF32, O::BufferStoreDword, O::SEndpgm}}; + return { + "VectorSinCosMaxFiniteSpecialCases", + code, + {}, + {0x00000000u, 0x3f800000u}, + {O::VMovB32, O::VSinF32, O::VCosF32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorCompareOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - std::vector results; - std::vector expected; - u32 dst = 10; + std::vector code; + std::vector results; + std::vector expected; + u32 dst = 10; - code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); - AppendVMovLiteral(&code, 2, 0x40000000u); - AppendVMovLiteral(&code, 3, 0x40400000u); - code.push_back(EncodeVop1(0x01, 4, InlineU32(1))); - AppendVMovLiteral(&code, 5, 0xffffffffu); + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + AppendVMovLiteral(&code, 2, 0x40000000u); + AppendVMovLiteral(&code, 3, 0x40400000u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(1))); + AppendVMovLiteral(&code, 5, 0xffffffffu); - auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) { - if (dst == 31u) { - dst++; - } - code.push_back(EncodeVopc(opcode, src0, src1)); - code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1)); - results.push_back(dst); - expected.push_back(value ? 1u : 0u); - dst++; - }; + auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) { + if (dst == 31u) { + dst++; + } + code.push_back(EncodeVopc(opcode, src0, src1)); + code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1)); + results.push_back(dst); + expected.push_back(value ? 1u : 0u); + dst++; + }; - append_compare(0x00, Vgpr(2), 3, false); - append_compare(0x01, Vgpr(2), 3, true); - append_compare(0x02, Vgpr(2), 3, false); - append_compare(0x03, Vgpr(2), 3, true); - append_compare(0x04, Vgpr(2), 3, false); - append_compare(0x05, Vgpr(2), 3, true); - append_compare(0x06, Vgpr(2), 3, false); - append_compare(0x07, Vgpr(2), 3, true); - append_compare(0x08, Vgpr(2), 3, false); - append_compare(0x09, Vgpr(2), 3, true); - append_compare(0x0a, Vgpr(2), 3, false); - append_compare(0x0b, Vgpr(2), 3, true); - append_compare(0x0c, Vgpr(2), 3, false); - append_compare(0x0d, Vgpr(2), 3, true); - append_compare(0x0e, Vgpr(2), 3, false); - append_compare(0x0f, Vgpr(2), 3, true); + append_compare(0x00, Vgpr(2), 3, false); + append_compare(0x01, Vgpr(2), 3, true); + append_compare(0x02, Vgpr(2), 3, false); + append_compare(0x03, Vgpr(2), 3, true); + append_compare(0x04, Vgpr(2), 3, false); + append_compare(0x05, Vgpr(2), 3, true); + append_compare(0x06, Vgpr(2), 3, false); + append_compare(0x07, Vgpr(2), 3, true); + append_compare(0x08, Vgpr(2), 3, false); + append_compare(0x09, Vgpr(2), 3, true); + append_compare(0x0a, Vgpr(2), 3, false); + append_compare(0x0b, Vgpr(2), 3, true); + append_compare(0x0c, Vgpr(2), 3, false); + append_compare(0x0d, Vgpr(2), 3, true); + append_compare(0x0e, Vgpr(2), 3, false); + append_compare(0x0f, Vgpr(2), 3, true); - append_compare(0x80, Vgpr(5), 4, false); - append_compare(0x81, Vgpr(5), 4, true); - append_compare(0x82, Vgpr(5), 4, false); - append_compare(0x83, Vgpr(5), 4, true); - append_compare(0x84, Vgpr(5), 4, false); - append_compare(0x85, Vgpr(5), 4, true); - append_compare(0x86, Vgpr(5), 4, false); - append_compare(0x87, Vgpr(5), 4, true); + append_compare(0x80, Vgpr(5), 4, false); + append_compare(0x81, Vgpr(5), 4, true); + append_compare(0x82, Vgpr(5), 4, false); + append_compare(0x83, Vgpr(5), 4, true); + append_compare(0x84, Vgpr(5), 4, false); + append_compare(0x85, Vgpr(5), 4, true); + append_compare(0x86, Vgpr(5), 4, false); + append_compare(0x87, Vgpr(5), 4, true); - append_compare(0xc0, Vgpr(4), 5, false); - append_compare(0xc1, Vgpr(4), 5, true); - append_compare(0xc2, Vgpr(4), 5, false); - append_compare(0xc3, Vgpr(4), 5, true); - append_compare(0xc4, Vgpr(4), 5, false); - append_compare(0xc5, Vgpr(4), 5, true); - append_compare(0xc6, Vgpr(4), 5, false); - append_compare(0xc7, Vgpr(4), 5, true); + append_compare(0xc0, Vgpr(4), 5, false); + append_compare(0xc1, Vgpr(4), 5, true); + append_compare(0xc2, Vgpr(4), 5, false); + append_compare(0xc3, Vgpr(4), 5, true); + append_compare(0xc4, Vgpr(4), 5, false); + append_compare(0xc5, Vgpr(4), 5, true); + append_compare(0xc6, Vgpr(4), 5, false); + append_compare(0xc7, Vgpr(4), 5, true); - code.push_back(0x7c1d00f9u); - code.push_back(0x86069201u); - code.push_back(EncodeVopc(0x84, 249, 4)); - code.push_back(EncodeVopcSdwa(5, 19, 1)); - code.push_back(EncodeVopc(0x01, 250, 3)); - code.push_back(EncodeVop2Dpp(2)); - AppendVMovLiteral(&code, 6, 0xc0000000u); - AppendVMovLiteral(&code, 7, 0x3f800000u); - AppendVop3(&code, 0x04, 20, Vgpr(6), Vgpr(7), 0, 1); - for (u32 i = 0; i < static_cast(results.size()); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendStoreSgpr(&code, 18, static_cast(results.size())); - expected.push_back(1u); - AppendStoreSgpr(&code, 19, static_cast(results.size() + 1u)); - expected.push_back(0u); - AppendStoreSgpr(&code, 106, static_cast(results.size() + 2u)); - expected.push_back(1u); - AppendStoreSgpr(&code, 20, static_cast(results.size() + 3u)); - expected.push_back(1u); - AppendEnd(&code); + code.push_back(0x7c1d00f9u); + code.push_back(0x86069201u); + code.push_back(EncodeVopc(0x84, 249, 4)); + code.push_back(EncodeVopcSdwa(5, 19, 1)); + code.push_back(EncodeVopc(0x01, 250, 3)); + code.push_back(EncodeVop2Dpp(2)); + AppendVMovLiteral(&code, 6, 0xc0000000u); + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendVop3(&code, 0x04, 20, Vgpr(6), Vgpr(7), 0, 1); + for (u32 i = 0; i < static_cast(results.size()); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendStoreSgpr(&code, 18, static_cast(results.size())); + expected.push_back(1u); + AppendStoreSgpr(&code, 19, static_cast(results.size() + 1u)); + expected.push_back(0u); + AppendStoreSgpr(&code, 106, static_cast(results.size() + 2u)); + expected.push_back(1u); + AppendStoreSgpr(&code, 20, static_cast(results.size() + 3u)); + expected.push_back(1u); + AppendEnd(&code); - return {"VectorCompareOps", - code, - {}, - expected, - {O::VMovB32, O::VCndmaskB32, O::VCmpFF32, O::VCmpLtF32, O::VCmpEqF32, - O::VCmpLeF32, O::VCmpGtF32, O::VCmpLgF32, O::VCmpGeF32, O::VCmpOF32, - O::VCmpUF32, O::VCmpNgeF32, O::VCmpNlgF32, O::VCmpNgtF32, O::VCmpNleF32, - O::VCmpNeqF32, O::VCmpNltF32, O::VCmpTruF32, O::VCmpFI32, O::VCmpLtI32, - O::VCmpEqI32, O::VCmpLeI32, O::VCmpGtI32, O::VCmpNeI32, O::VCmpGeI32, - O::VCmpTI32, O::VCmpFU32, O::VCmpLtU32, O::VCmpEqU32, O::VCmpLeU32, - O::VCmpGtU32, O::VCmpNeU32, O::VCmpGeU32, O::VCmpTU32, O::BufferStoreDword, - O::SEndpgm}}; + return {"VectorCompareOps", + code, + {}, + expected, + {O::VMovB32, O::VCndmaskB32, O::VCmpFF32, O::VCmpLtF32, + O::VCmpEqF32, O::VCmpLeF32, O::VCmpGtF32, O::VCmpLgF32, + O::VCmpGeF32, O::VCmpOF32, O::VCmpUF32, O::VCmpNgeF32, + O::VCmpNlgF32, O::VCmpNgtF32, O::VCmpNleF32, O::VCmpNeqF32, + O::VCmpNltF32, O::VCmpTruF32, O::VCmpFI32, O::VCmpLtI32, + O::VCmpEqI32, O::VCmpLeI32, O::VCmpGtI32, O::VCmpNeI32, + O::VCmpGeI32, O::VCmpTI32, O::VCmpFU32, O::VCmpLtU32, + O::VCmpEqU32, O::VCmpLeU32, O::VCmpGtU32, O::VCmpNeU32, + O::VCmpGeU32, O::VCmpTU32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVop3CompareNeU64OnGpu() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + std::vector code; + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); - code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec - code.push_back(0xd4e5006au); - code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc - code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); - AppendStoreVgpr(&code, 2, 0); + code.push_back(EncodeSop1(0x04, 106, 126)); // s_mov_b64 vcc, exec + code.push_back(0xd4e5006au); + code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc + code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); + AppendStoreVgpr(&code, 2, 0); - AppendSMovLiteral(&code, 106, 0); - AppendSMovLiteral(&code, 107, 0); - code.push_back(0xd4e5006au); - code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc - code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); - AppendStoreVgpr(&code, 3, 1); - AppendEnd(&code); + AppendSMovLiteral(&code, 106, 0); + AppendSMovLiteral(&code, 107, 0); + code.push_back(0xd4e5006au); + code.push_back(0x0000d47eu); // v_cmp_ne_u64 vcc, exec, vcc + code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); + AppendStoreVgpr(&code, 3, 1); + AppendEnd(&code); - return {"VectorVop3CompareNeU64OnGpu", - code, - {}, - {0, 1}, - {O::VMovB32, O::SMovB64, O::VCmpNeU64, O::VCndmaskB32, O::SMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"VectorVop3CompareNeU64OnGpu", + code, + {}, + {0, 1}, + {O::VMovB32, O::SMovB64, O::VCmpNeU64, O::VCndmaskB32, O::SMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVop3CompareEqI64OnGpu() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); - code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec + std::vector code; + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + code.push_back(EncodeSop1(0x04, 4, 126)); // s_mov_b64 s[4:5], exec - code.push_back(0xd4a2006au); - code.push_back(0x0000087eu); // v_cmp_eq_i64 vcc, exec, s[4:5] - code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); - AppendStoreVgpr(&code, 2, 0); + code.push_back(0xd4a2006au); + code.push_back(0x0000087eu); // v_cmp_eq_i64 vcc, exec, s[4:5] + code.push_back(EncodeVop2(0x01, 2, InlineU32(0), 1)); + AppendStoreVgpr(&code, 2, 0); - AppendSMovLiteral(&code, 4, 0); - code.push_back(0xd4a2006au); - code.push_back(0x0000087eu); // low dword mismatch - code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); - AppendStoreVgpr(&code, 3, 1); - AppendEnd(&code); + AppendSMovLiteral(&code, 4, 0); + code.push_back(0xd4a2006au); + code.push_back(0x0000087eu); // low dword mismatch + code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); + AppendStoreVgpr(&code, 3, 1); + AppendEnd(&code); - return {"VectorVop3CompareEqI64OnGpu", - code, - {}, - {1, 0}, - {O::VMovB32, O::SMovB64, O::VCmpEqI64, O::VCndmaskB32, O::SMovB32, - O::BufferStoreDword, O::SEndpgm}}; + return {"VectorVop3CompareEqI64OnGpu", + code, + {}, + {1, 0}, + {O::VMovB32, O::SMovB64, O::VCmpEqI64, O::VCndmaskB32, O::SMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorCompareClassF32() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - std::vector expected; - AppendVMovU32(&code, 1, 1); + std::vector code; + std::vector expected; + AppendVMovU32(&code, 1, 1); - u32 out = 0; - auto append_class = [&](u32 bits, u32 class_mask, bool match) { - AppendVMovLiteral(&code, 0, bits); - AppendVMovU32(&code, 2, class_mask); - code.push_back(EncodeVopc(0x88, Vgpr(0), 2)); - code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); - AppendStoreVgpr(&code, 3, out++); - expected.push_back(match ? 1u : 0u); - }; + u32 out = 0; + auto append_class = [&](u32 bits, u32 class_mask, bool match) { + AppendVMovLiteral(&code, 0, bits); + AppendVMovU32(&code, 2, class_mask); + code.push_back(EncodeVopc(0x88, Vgpr(0), 2)); + code.push_back(EncodeVop2(0x01, 3, InlineU32(0), 1)); + AppendStoreVgpr(&code, 3, out++); + expected.push_back(match ? 1u : 0u); + }; - append_class(0x7fc00000u, 1u << 1u, true); - append_class(0xff800000u, 1u << 2u, true); - append_class(0xbf800000u, 1u << 3u, true); - append_class(0x80000000u, 1u << 5u, true); - append_class(0x00000000u, 1u << 6u, true); - append_class(0x40000000u, 1u << 8u, true); - append_class(0x7f800000u, 1u << 9u, true); - append_class(0x40000000u, 1u << 3u, false); + append_class(0x7fc00000u, 1u << 1u, true); + append_class(0xff800000u, 1u << 2u, true); + append_class(0xbf800000u, 1u << 3u, true); + append_class(0x80000000u, 1u << 5u, true); + append_class(0x00000000u, 1u << 6u, true); + append_class(0x40000000u, 1u << 8u, true); + append_class(0x7f800000u, 1u << 9u, true); + append_class(0x40000000u, 1u << 3u, false); - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovU32(&code, 2, 1u << 3u); - AppendVop3(&code, 0x88, 20, Vgpr(0), Vgpr(2), 0, 0, 0, false, 0, 1); - AppendStoreSgprPair(&code, 20, out); - expected.push_back(1u); - expected.push_back(0u); - AppendEnd(&code); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovU32(&code, 2, 1u << 3u); + AppendVop3(&code, 0x88, 20, Vgpr(0), Vgpr(2), 0, 0, 0, false, 0, 1); + AppendStoreSgprPair(&code, 20, out); + expected.push_back(1u); + expected.push_back(0u); + AppendEnd(&code); - return {"VectorCompareClassF32", - code, - {}, - expected, - {O::VMovB32, O::VCmpClassF32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorCompareClassF32", + code, + {}, + expected, + {O::VMovB32, O::VCmpClassF32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase VectorCompareF16Ops() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - std::vector expected; - AppendVMovU32(&code, 1, 1); - AppendVMovLiteral(&code, 2, 0x40003c00u); // low=1.0h, high=2.0h - AppendVMovLiteral(&code, 3, 0x44004200u); // low=3.0h, high=4.0h - AppendVMovLiteral(&code, 4, 0x00007e00u); // low=qNaN + std::vector code; + std::vector expected; + AppendVMovU32(&code, 1, 1); + AppendVMovLiteral(&code, 2, 0x40003c00u); // low=1.0h, high=2.0h + AppendVMovLiteral(&code, 3, 0x44004200u); // low=3.0h, high=4.0h + AppendVMovLiteral(&code, 4, 0x00007e00u); // low=qNaN - u32 dst = 10; - u32 out = 0; - auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) { - code.push_back(EncodeVopc(opcode, src0, src1)); - code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1)); - AppendStoreVgpr(&code, dst, out++); - expected.push_back(value ? 1u : 0u); - dst++; - }; + u32 dst = 10; + u32 out = 0; + auto append_compare = [&](u32 opcode, u32 src0, u32 src1, bool value) { + code.push_back(EncodeVopc(opcode, src0, src1)); + code.push_back(EncodeVop2(0x01, dst, InlineU32(0), 1)); + AppendStoreVgpr(&code, dst, out++); + expected.push_back(value ? 1u : 0u); + dst++; + }; - append_compare(0xc9, Vgpr(2), 3, true); - append_compare(0xca, Vgpr(2), 2, true); - append_compare(0xcb, Vgpr(2), 3, true); - append_compare(0xcc, Vgpr(3), 2, true); - append_compare(0xcd, Vgpr(2), 3, true); - append_compare(0xce, Vgpr(3), 2, true); - append_compare(0xed, Vgpr(4), 2, true); + append_compare(0xc9, Vgpr(2), 3, true); + append_compare(0xca, Vgpr(2), 2, true); + append_compare(0xcb, Vgpr(2), 3, true); + append_compare(0xcc, Vgpr(3), 2, true); + append_compare(0xcd, Vgpr(2), 3, true); + append_compare(0xce, Vgpr(3), 2, true); + append_compare(0xed, Vgpr(4), 2, true); - auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeVopc(opcode, src0, src1)); - AppendStoreVgpr(&code, 1, out++); - expected.push_back(1u); - }; + auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeVopc(opcode, src0, src1)); + AppendStoreVgpr(&code, 1, out++); + expected.push_back(1u); + }; - append_cmpx(0xd9, Vgpr(2), 3); - append_cmpx(0xda, Vgpr(2), 2); - append_cmpx(0xdb, Vgpr(2), 3); - append_cmpx(0xdc, Vgpr(3), 2); - append_cmpx(0xde, Vgpr(3), 2); - append_cmpx(0xfd, Vgpr(4), 2); - append_cmpx(0xfe, Vgpr(2), 2); - AppendEnd(&code); + append_cmpx(0xd9, Vgpr(2), 3); + append_cmpx(0xda, Vgpr(2), 2); + append_cmpx(0xdb, Vgpr(2), 3); + append_cmpx(0xdc, Vgpr(3), 2); + append_cmpx(0xde, Vgpr(3), 2); + append_cmpx(0xfd, Vgpr(4), 2); + append_cmpx(0xfe, Vgpr(2), 2); + AppendEnd(&code); - return {"VectorCompareF16Ops", - code, - {}, - expected, - {O::VMovB32, O::SMovB32, O::VCmpLtF16, O::VCmpEqF16, O::VCmpLeF16, O::VCmpGtF16, - O::VCmpLgF16, O::VCmpGeF16, O::VCmpNeqF16, O::VCmpxLtF16, O::VCmpxEqF16, O::VCmpxLeF16, - O::VCmpxGtF16, O::VCmpxGeF16, O::VCmpxNeqF16, O::VCmpxNltF16, O::VCndmaskB32, - O::BufferStoreDword, O::SEndpgm}}; + return {"VectorCompareF16Ops", + code, + {}, + expected, + {O::VMovB32, O::SMovB32, O::VCmpLtF16, O::VCmpEqF16, O::VCmpLeF16, + O::VCmpGtF16, O::VCmpLgF16, O::VCmpGeF16, O::VCmpNeqF16, + O::VCmpxLtF16, O::VCmpxEqF16, O::VCmpxLeF16, O::VCmpxGtF16, + O::VCmpxGeF16, O::VCmpxNeqF16, O::VCmpxNltF16, O::VCndmaskB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase Vop2SdwaCndmaskSourceModifier() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 6, 0x00000000u); - AppendVMovLiteral(&code, 47, 0x12345678u); - AppendVMovLiteral(&code, 53, 0x3f800000u); - code.push_back(EncodeVopc(0xc7, Vgpr(6), 6)); // v_cmp_t_u32 - code.push_back(0x025e6af9u); - code.push_back(0x16060635u); - AppendStoreVgpr(&code, 47, 0); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 6, 0x00000000u); + AppendVMovLiteral(&code, 47, 0x12345678u); + AppendVMovLiteral(&code, 53, 0x3f800000u); + code.push_back(EncodeVopc(0xc7, Vgpr(6), 6)); // v_cmp_t_u32 + code.push_back(0x025e6af9u); + code.push_back(0x16060635u); + AppendStoreVgpr(&code, 47, 0); + AppendEnd(&code); - return {"Vop2SdwaCndmaskSourceModifier", - code, - {}, - {0xbf800000u}, - {O::VMovB32, O::VCmpTU32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; + return {"Vop2SdwaCndmaskSourceModifier", + code, + {}, + {0xbf800000u}, + {O::VMovB32, O::VCmpTU32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase Vop2SdwaCndmaskFullDestinationWithSubDwordSource() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xabcd1234u); - AppendVMovLiteral(&code, 1, 0x55667788u); - code.push_back(EncodeVopc(0xc0, Vgpr(0), 0)); // v_cmp_f_u32 - code.push_back(0x020e02f9u); - code.push_back(0x06040600u); - AppendStoreVgpr(&code, 7, 0); - code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 - code.push_back(0x020e02f9u); - code.push_back(0x06040600u); - AppendStoreVgpr(&code, 7, 1); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0xabcd1234u); + AppendVMovLiteral(&code, 1, 0x55667788u); + code.push_back(EncodeVopc(0xc0, Vgpr(0), 0)); // v_cmp_f_u32 + code.push_back(0x020e02f9u); + code.push_back(0x06040600u); + AppendStoreVgpr(&code, 7, 0); + code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 + code.push_back(0x020e02f9u); + code.push_back(0x06040600u); + AppendStoreVgpr(&code, 7, 1); + AppendEnd(&code); - TestCase test; - test.name = "Vop2SdwaCndmaskFullDestinationWithSubDwordSource"; - test.code = code; - test.expected = {0x00001234u, 0x55667788u}; - test.opcodes = {O::VMovB32, O::VCmpFU32, O::VCmpTU32, O::VCndmaskB32, - O::BufferStoreDword, O::SEndpgm}; - test.required_spirv = {"OpBitFieldUExtract", "OpSelect"}; - return test; + TestCase test; + test.name = "Vop2SdwaCndmaskFullDestinationWithSubDwordSource"; + test.code = code; + test.expected = {0x00001234u, 0x55667788u}; + test.opcodes = {O::VMovB32, O::VCmpFU32, O::VCmpTU32, + O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpSelect"}; + return test; } TestCase Vop3CndmaskUsesSgprMaskLaneBits() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVopc(0xc0, Vgpr(0), 0), - }; - AppendSMovLiteral(&code, 4, 1); - AppendSMovLiteral(&code, 5, 0); - AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4); - code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1)); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVopc(0xc0, Vgpr(0), 0), + }; + AppendSMovLiteral(&code, 4, 1); + AppendSMovLiteral(&code, 5, 0); + AppendVop3(&code, 0x101, 2, InlineU32(10), InlineU32(20), 4); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 1)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "Vop3CndmaskUsesSgprMaskLaneBits"; - test.code = code; - test.expected = {20, 10, 10, 10, 20, 10, 10, 10}; - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, - O::SMovB32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "Vop3CndmaskUsesSgprMaskLaneBits"; + test.code = code; + test.expected = {20, 10, 10, 10, 20, 10, 10, 10}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, + O::SMovB32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase Vop3CndmaskAllowsDataSourceModifier() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 7, 0x3f800000u); - AppendSMovLiteral(&code, 34, 1); - AppendSMovLiteral(&code, 35, 0); - code.push_back(0xd5010004u); - code.push_back(0x408a0f07u); - AppendStoreVgpr(&code, 4, 0); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 7, 0x3f800000u); + AppendSMovLiteral(&code, 34, 1); + AppendSMovLiteral(&code, 35, 0); + code.push_back(0xd5010004u); + code.push_back(0x408a0f07u); + AppendStoreVgpr(&code, 4, 0); + AppendEnd(&code); - return {"Vop3CndmaskAllowsDataSourceModifier", - code, - {}, - {0xbf800000u}, - {O::VMovB32, O::SMovB32, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; + return {"Vop3CndmaskAllowsDataSourceModifier", + code, + {}, + {0xbf800000u}, + {O::VMovB32, O::SMovB32, O::VCndmaskB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase VectorCompareExecOps() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); - AppendVMovLiteral(&code, 2, 0x40000000u); - AppendVMovLiteral(&code, 3, 0x40400000u); - code.push_back(EncodeVop1(0x01, 4, InlineU32(1))); - AppendVMovLiteral(&code, 5, 0xffffffffu); + std::vector code; + code.push_back(EncodeVop1(0x01, 1, InlineU32(1))); + AppendVMovLiteral(&code, 2, 0x40000000u); + AppendVMovLiteral(&code, 3, 0x40400000u); + code.push_back(EncodeVop1(0x01, 4, InlineU32(1))); + AppendVMovLiteral(&code, 5, 0xffffffffu); - u32 out = 0; - auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeVopc(opcode, src0, src1)); - AppendStoreVgpr(&code, 1, out++); - }; + u32 out = 0; + auto append_cmpx = [&](u32 opcode, u32 src0, u32 src1) { + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeVopc(opcode, src0, src1)); + AppendStoreVgpr(&code, 1, out++); + }; - append_cmpx(0x11, Vgpr(2), 3); - append_cmpx(0x12, Vgpr(3), 3); - append_cmpx(0x13, Vgpr(2), 3); - append_cmpx(0x14, Vgpr(3), 2); - append_cmpx(0x15, Vgpr(2), 3); - append_cmpx(0x16, Vgpr(3), 3); - append_cmpx(0x19, Vgpr(2), 3); - append_cmpx(0x1a, Vgpr(3), 3); - append_cmpx(0x1b, Vgpr(2), 3); - append_cmpx(0x1c, Vgpr(3), 2); - append_cmpx(0x1d, Vgpr(2), 3); - append_cmpx(0x1e, Vgpr(3), 3); - append_cmpx(0x91, Vgpr(5), 4); - append_cmpx(0x92, Vgpr(4), 4); - append_cmpx(0x93, Vgpr(5), 4); - append_cmpx(0x94, Vgpr(4), 5); - append_cmpx(0x95, Vgpr(5), 4); - append_cmpx(0x96, Vgpr(4), 4); - append_cmpx(0xd1, Vgpr(4), 5); - append_cmpx(0xd2, Vgpr(4), 4); - append_cmpx(0xd3, Vgpr(4), 5); - append_cmpx(0xd4, Vgpr(5), 4); - append_cmpx(0xd5, Vgpr(4), 5); - append_cmpx(0xd6, Vgpr(4), 4); - AppendEnd(&code); + append_cmpx(0x11, Vgpr(2), 3); + append_cmpx(0x12, Vgpr(3), 3); + append_cmpx(0x13, Vgpr(2), 3); + append_cmpx(0x14, Vgpr(3), 2); + append_cmpx(0x15, Vgpr(2), 3); + append_cmpx(0x16, Vgpr(3), 3); + append_cmpx(0x19, Vgpr(2), 3); + append_cmpx(0x1a, Vgpr(3), 3); + append_cmpx(0x1b, Vgpr(2), 3); + append_cmpx(0x1c, Vgpr(3), 2); + append_cmpx(0x1d, Vgpr(2), 3); + append_cmpx(0x1e, Vgpr(3), 3); + append_cmpx(0x91, Vgpr(5), 4); + append_cmpx(0x92, Vgpr(4), 4); + append_cmpx(0x93, Vgpr(5), 4); + append_cmpx(0x94, Vgpr(4), 5); + append_cmpx(0x95, Vgpr(5), 4); + append_cmpx(0x96, Vgpr(4), 4); + append_cmpx(0xd1, Vgpr(4), 5); + append_cmpx(0xd2, Vgpr(4), 4); + append_cmpx(0xd3, Vgpr(4), 5); + append_cmpx(0xd4, Vgpr(5), 4); + append_cmpx(0xd5, Vgpr(4), 5); + append_cmpx(0xd6, Vgpr(4), 4); + AppendEnd(&code); - return {"VectorCompareExecOps", - code, - {}, - std::vector(out, 1), - {O::SMovB32, O::VMovB32, O::VCmpxLtF32, O::VCmpxEqF32, O::VCmpxLeF32, - O::VCmpxGtF32, O::VCmpxLgF32, O::VCmpxGeF32, O::VCmpxNgeF32, O::VCmpxNlgF32, - O::VCmpxNgtF32, O::VCmpxNleF32, O::VCmpxNeqF32, O::VCmpxNltF32, O::VCmpxLtI32, - O::VCmpxEqI32, O::VCmpxLeI32, O::VCmpxGtI32, O::VCmpxNeI32, O::VCmpxGeI32, - O::VCmpxLtU32, O::VCmpxEqU32, O::VCmpxLeU32, O::VCmpxGtU32, O::VCmpxNeU32, - O::VCmpxGeU32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorCompareExecOps", + code, + {}, + std::vector(out, 1), + {O::SMovB32, O::VMovB32, O::VCmpxLtF32, O::VCmpxEqF32, + O::VCmpxLeF32, O::VCmpxGtF32, O::VCmpxLgF32, O::VCmpxGeF32, + O::VCmpxNgeF32, O::VCmpxNlgF32, O::VCmpxNgtF32, O::VCmpxNleF32, + O::VCmpxNeqF32, O::VCmpxNltF32, O::VCmpxLtI32, O::VCmpxEqI32, + O::VCmpxLeI32, O::VCmpxGtI32, O::VCmpxNeI32, O::VCmpxGeI32, + O::VCmpxLtU32, O::VCmpxEqU32, O::VCmpxLeU32, O::VCmpxGtU32, + O::VCmpxNeU32, O::VCmpxGeU32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVop3FloatCompareNegSourceModifier() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovU32(&code, 1, 0); - AppendVop3(&code, 0x01, 20, Vgpr(0), Vgpr(1), 0, 0, 0, false, 0, - 0x1); // -1.0 < 0.0 - AppendStoreSgprPair(&code, 20, 0); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovU32(&code, 1, 0); + AppendVop3(&code, 0x01, 20, Vgpr(0), Vgpr(1), 0, 0, 0, false, 0, + 0x1); // -1.0 < 0.0 + AppendStoreSgprPair(&code, 20, 0); + AppendEnd(&code); - return {"VectorVop3FloatCompareNegSourceModifier", - code, - {}, - {1, 0}, - {O::VMovB32, O::VCmpLtF32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorVop3FloatCompareNegSourceModifier", + code, + {}, + {1, 0}, + {O::VMovB32, O::VCmpLtF32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVop3CmpxWritesExecMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 0, 2); - AppendVMovU32(&code, 1, 1); - AppendVMovU32(&code, 2, 0); - AppendVMovU32(&code, 30, 0); - AppendVop3(&code, 0xd1, 5, Vgpr(0), Vgpr(1)); // v_cmpx_lt_u32, false - AppendVMovU32(&code, 2, 7); - AppendBufferStoreDword(&code, 2, 30); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 0, 2); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 30, 0); + AppendVop3(&code, 0xd1, 5, Vgpr(0), Vgpr(1)); // v_cmpx_lt_u32, false + AppendVMovU32(&code, 2, 7); + AppendBufferStoreDword(&code, 2, 30); + AppendEnd(&code); - return {"VectorVop3CmpxWritesExecMask", - code, - {0}, - {0}, - {O::VMovB32, O::VCmpxLtU32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorVop3CmpxWritesExecMask", + code, + {0}, + {0}, + {O::VMovB32, O::VCmpxLtU32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorVopcSdwaCmpxWritesExecMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 0, 2); - AppendVMovU32(&code, 1, 1); - AppendVMovU32(&code, 2, 0); - AppendVMovU32(&code, 30, 0); - code.push_back(EncodeVopc(0xd1, 249, 1)); // v_cmpx_lt_u32.sdwa, false - code.push_back(EncodeVopcSdwa(0)); - AppendVMovU32(&code, 2, 7); - AppendBufferStoreDword(&code, 2, 30); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 0, 2); + AppendVMovU32(&code, 1, 1); + AppendVMovU32(&code, 2, 0); + AppendVMovU32(&code, 30, 0); + code.push_back(EncodeVopc(0xd1, 249, 1)); // v_cmpx_lt_u32.sdwa, false + code.push_back(EncodeVopcSdwa(0)); + AppendVMovU32(&code, 2, 7); + AppendBufferStoreDword(&code, 2, 30); + AppendEnd(&code); - return {"VectorVopcSdwaCmpxWritesExecMask", - code, - {0}, - {0}, - {O::VMovB32, O::VCmpxLtU32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorVopcSdwaCmpxWritesExecMask", + code, + {0}, + {0}, + {O::VMovB32, O::VCmpxLtU32, O::BufferStoreDword, O::SEndpgm}}; } TestCase VectorCompareInvertedMaskSelect() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x12345678u); - code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 v0, v0 - code.push_back(EncodeSop1(0x08, 106, 106)); // s_not_b64 vcc, vcc - code.push_back(EncodeVop2(0x01, 1, Vgpr(0), 128)); // v_cndmask_b32 v1, v0, 0 - AppendBufferStoreDword(&code, 1, 30); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeVopc(0xc7, Vgpr(0), 0)); // v_cmp_t_u32 v0, v0 + code.push_back(EncodeSop1(0x08, 106, 106)); // s_not_b64 vcc, vcc + code.push_back(EncodeVop2(0x01, 1, Vgpr(0), 128)); // v_cndmask_b32 v1, v0, 0 + AppendBufferStoreDword(&code, 1, 30); + AppendEnd(&code); - return {"VectorCompareInvertedMaskSelect", - code, - {}, - {0x12345678u}, - {O::VMovB32, O::VCmpTU32, O::SNotB64, O::VCndmaskB32, O::BufferStoreDword, O::SEndpgm}}; + return {"VectorCompareInvertedMaskSelect", + code, + {}, + {0x12345678u}, + {O::VMovB32, O::VCmpTU32, O::SNotB64, O::VCndmaskB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase BranchSelect() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeSMovB32(0, InlineU32(0)), - EncodeSopc(0x06, 0, InlineU32(0)), - EncodeSopp(0x05, 2), - EncodeSMovB32(1, InlineU32(1)), - EncodeSopp(0x02, 1), - EncodeSMovB32(1, InlineU32(7)), - EncodeSop2(0x0a, 2, 1, InlineU32(2)), - EncodeVop1(0x01, 0, 2), - }; - AppendBufferStoreDword(&code, 0, 30); - AppendEnd(&code); - return {"BranchSelect", - code, - {}, - {7}, - {O::SMovB32, O::SCmpEqU32, O::SCbranchScc1, O::SBranch, O::SCselectB32, O::VMovB32, - O::BufferStoreDword, O::SEndpgm}}; + std::vector code = { + EncodeSMovB32(0, InlineU32(0)), + EncodeSopc(0x06, 0, InlineU32(0)), + EncodeSopp(0x05, 2), + EncodeSMovB32(1, InlineU32(1)), + EncodeSopp(0x02, 1), + EncodeSMovB32(1, InlineU32(7)), + EncodeSop2(0x0a, 2, 1, InlineU32(2)), + EncodeVop1(0x01, 0, 2), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"BranchSelect", + code, + {}, + {7}, + {O::SMovB32, O::SCmpEqU32, O::SCbranchScc1, O::SBranch, + O::SCselectB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase SimpleLoop() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeSMovB32(0, InlineU32(0)), - EncodeSopc(0x0a, 0, InlineU32(4)), - EncodeSopp(0x04, 2), - EncodeSop2(0x00, 0, 0, InlineU32(1)), - EncodeSopp(0x02, 0xfffcu), - EncodeVop1(0x01, 0, 0), - }; - AppendBufferStoreDword(&code, 0, 30); - AppendEnd(&code); - return {"SimpleLoop", - code, - {}, - {4}, - {O::SMovB32, O::SCmpLtU32, O::SCbranchScc0, O::SAddU32, O::SBranch, O::VMovB32, - O::BufferStoreDword, O::SEndpgm}}; + std::vector code = { + EncodeSMovB32(0, InlineU32(0)), + EncodeSopc(0x0a, 0, InlineU32(4)), + EncodeSopp(0x04, 2), + EncodeSop2(0x00, 0, 0, InlineU32(1)), + EncodeSopp(0x02, 0xfffcu), + EncodeVop1(0x01, 0, 0), + }; + AppendBufferStoreDword(&code, 0, 30); + AppendEnd(&code); + return {"SimpleLoop", + code, + {}, + {4}, + {O::SMovB32, O::SCmpLtU32, O::SCbranchScc0, O::SAddU32, O::SBranch, + O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase BranchVccnzUsesWholeMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVop2(0x1a, 3, InlineU32(2), 1), - EncodeVopc(0xc2, InlineU32(0), 0), - EncodeSopp(0x07, 2), - EncodeVop1(0x01, 2, InlineU32(11)), - EncodeSopp(0x02, 1), - }; - AppendVMovU32(&code, 2, 42); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + EncodeVopc(0xc2, InlineU32(0), 0), + EncodeSopp(0x07, 2), + EncodeVop1(0x01, 2, InlineU32(11)), + EncodeSopp(0x02, 1), + }; + AppendVMovU32(&code, 2, 42); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "BranchVccnzUsesWholeMask"; - test.code = code; - test.expected = std::vector(8, 42); - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpEqU32, - O::SCbranchVccnz, O::SBranch, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "BranchVccnzUsesWholeMask"; + test.code = code; + test.expected = std::vector(8, 42); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VCmpEqU32, O::SCbranchVccnz, O::SBranch, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase ScalarMemoryLoadVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - struct Load { - u32 opcode; - u32 dst; - u32 byte_offset; - u32 dwords; - }; + struct Load { + u32 opcode; + u32 dst; + u32 byte_offset; + u32 dwords; + }; - const Load s_loads[] = { - {0x00, 20, 0, 1}, {0x01, 21, 4, 2}, {0x02, 23, 12, 4}, - {0x03, 27, 28, 8}, {0x04, 40, 60, 16}, - }; - const Load s_buffer_loads[] = { - {0x08, 56, 0, 1}, {0x09, 57, 4, 2}, {0x0a, 59, 12, 4}, - {0x0b, 63, 28, 8}, {0x0c, 71, 60, 16}, - }; + const Load s_loads[] = { + {0x00, 20, 0, 1}, {0x01, 21, 4, 2}, {0x02, 23, 12, 4}, + {0x03, 27, 28, 8}, {0x04, 40, 60, 16}, + }; + const Load s_buffer_loads[] = { + {0x08, 56, 0, 1}, {0x09, 57, 4, 2}, {0x0a, 59, 12, 4}, + {0x0b, 63, 28, 8}, {0x0c, 71, 60, 16}, + }; - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(0))); - for (const auto& load: s_loads) { - AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset); - } - for (const auto& load: s_buffer_loads) { - AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset); - } + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + for (const auto &load : s_loads) { + AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset); + } + for (const auto &load : s_buffer_loads) { + AppendSmemLoadOpcode(&code, load.opcode, load.dst, load.byte_offset); + } - u32 out = 0; - for (const auto& load: s_loads) { - for (u32 i = 0; i < load.dwords; i++) { - AppendStoreSgpr(&code, load.dst + i, out++); - } - } - for (const auto& load: s_buffer_loads) { - for (u32 i = 0; i < load.dwords; i++) { - AppendStoreSgpr(&code, load.dst + i, out++); - } - } - AppendEnd(&code); + u32 out = 0; + for (const auto &load : s_loads) { + for (u32 i = 0; i < load.dwords; i++) { + AppendStoreSgpr(&code, load.dst + i, out++); + } + } + for (const auto &load : s_buffer_loads) { + for (u32 i = 0; i < load.dwords; i++) { + AppendStoreSgpr(&code, load.dst + i, out++); + } + } + AppendEnd(&code); - std::vector initial; - for (u32 i = 0; i < 31u; i++) { - initial.push_back(0x10000000u + i); - } - std::vector expected = initial; - expected.insert(expected.end(), initial.begin(), initial.end()); + std::vector initial; + for (u32 i = 0; i < 31u; i++) { + initial.push_back(0x10000000u + i); + } + std::vector expected = initial; + expected.insert(expected.end(), initial.begin(), initial.end()); - return {"ScalarMemoryLoadVariants", - code, - initial, - expected, - {O::SMovB32, O::SLoadDword, O::SLoadDwordx2, O::SLoadDwordx4, O::SLoadDwordx8, - O::SLoadDwordx16, O::SBufferLoadDword, O::SBufferLoadDwordx2, O::SBufferLoadDwordx4, - O::SBufferLoadDwordx8, O::SBufferLoadDwordx16, O::VMovB32, O::BufferStoreDword, - O::SEndpgm}}; + return {"ScalarMemoryLoadVariants", + code, + initial, + expected, + {O::SMovB32, O::SLoadDword, O::SLoadDwordx2, O::SLoadDwordx4, + O::SLoadDwordx8, O::SLoadDwordx16, O::SBufferLoadDword, + O::SBufferLoadDwordx2, O::SBufferLoadDwordx4, O::SBufferLoadDwordx8, + O::SBufferLoadDwordx16, O::VMovB32, O::BufferStoreDword, + O::SEndpgm}}; } TestCase ScalarLoadSignedImmediateOffsetAddsSoffset() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(8))); - code.push_back(EncodeSMovB32(2, InlineU32(0))); - code.push_back(EncodeSMovB32(3, InlineU32(0))); - code.push_back(EncodeSmem0(0x00, 1, 1)); - code.push_back(EncodeSmem1(0x1ffffcu, 0)); - AppendStoreSgpr(&code, 1, 0); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(8))); + code.push_back(EncodeSMovB32(2, InlineU32(0))); + code.push_back(EncodeSMovB32(3, InlineU32(0))); + code.push_back(EncodeSmem0(0x00, 1, 1)); + code.push_back(EncodeSmem1(0x1ffffcu, 0)); + AppendStoreSgpr(&code, 1, 0); + AppendEnd(&code); - return {"ScalarLoadSignedImmediateOffsetAddsSoffset", - code, - {0x11111111u, 0x22222222u}, - {0x22222222u, 0x22222222u}, - {O::SMovB32, O::SLoadDword, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarLoadSignedImmediateOffsetAddsSoffset", + code, + {0x11111111u, 0x22222222u}, + {0x22222222u, 0x22222222u}, + {O::SMovB32, O::SLoadDword, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferLoadStore() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendBufferLoadDword(&code, 0, 30); - code.push_back(EncodeVop1(0x01, 31, InlineU32(4))); - AppendBufferStoreDword(&code, 0, 31); - AppendEnd(&code); - return {"BufferLoadStore", - code, - {0x11223344u, 0}, - {0x11223344u, 0x11223344u}, - {O::BufferLoadDword, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + std::vector code; + AppendBufferLoadDword(&code, 0, 30); + code.push_back(EncodeVop1(0x01, 31, InlineU32(4))); + AppendBufferStoreDword(&code, 0, 31); + AppendEnd(&code); + return {"BufferLoadStore", + code, + {0x11223344u, 0}, + {0x11223344u, 0x11223344u}, + {O::BufferLoadDword, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 21, 8); - code.push_back(EncodeMubuf0(0x0cu, 0, true, true)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 8); + code.push_back(EncodeMubuf0(0x0cu, 0, true, true)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - return {"BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset", - code, - {0x11111111u, 0x22222222u, 0x33333333u}, - {0x33333333u, 0x22222222u, 0x33333333u}, - {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}}; + return {"BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset", + code, + {0x11111111u, 0x22222222u, 0x33333333u}, + {0x33333333u, 0x22222222u, 0x33333333u}, + {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 21, 12); - AppendVMovLiteral(&code, 0, 0xabcdef01u); - code.push_back(EncodeMubuf0(0x1cu, 0, true, true)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 12); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x1cu, 0, true, true)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - return {"BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset", - code, - {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, - {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u}, - {O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, + {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u}, + {O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferLoadDwordNoAddressFlagsIgnoresVaddr() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 8); - code.push_back(EncodeMubuf0(0x0cu, 0, false, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendStoreVgpr(&code, 0, 3); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeMubuf0(0x0cu, 0, false, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 3); + AppendEnd(&code); - return {"BufferLoadDwordNoAddressFlagsIgnoresVaddr", - code, - {0x11111111u, 0x22222222u, 0x33333333u, 0}, - {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u}, - {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}}; + return {"BufferLoadDwordNoAddressFlagsIgnoresVaddr", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0}, + {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u}, + {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferLoadDwordIdxenUsesDescriptorStride() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - code.push_back(EncodeMubuf0(0x0cu, 0, true, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + code.push_back(EncodeMubuf0(0x0cu, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadDwordIdxenUsesDescriptorStride"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, - 0x66666666u, 0x77777777u, 0x88888888u, 0x99aabbccu}; - test.expected = {0x99aabbccu, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, - 0x66666666u, 0x77777777u, 0x88888888u, 0x99aabbccu}; - test.opcodes = {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(16, 3); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadDwordIdxenUsesDescriptorStride"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0x99aabbccu}; + test.expected = {0x99aabbccu, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0x99aabbccu}; + test.opcodes = {O::VMovB32, O::BufferLoadDword, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 3); + test.has_user_data = true; + return test; } TestCase BufferStoreDwordIdxenUsesDescriptorStride() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovLiteral(&code, 0, 0xabcdef01u); - code.push_back(EncodeMubuf0(0x1cu, 0, true, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x1cu, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreDwordIdxenUsesDescriptorStride"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, - 0x66666666u, 0x77777777u, 0x88888888u, 0x99aabbccu}; - test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, - 0x66666666u, 0x77777777u, 0x88888888u, 0xabcdef01u}; - test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(16, 3); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreDwordIdxenUsesDescriptorStride"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0x99aabbccu}; + test.expected = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, + 0x77777777u, 0x88888888u, 0xabcdef01u}; + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 3); + test.has_user_data = true; + return test; } TestCase BufferStoreDwordAppliesHostOffset() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0xabcdef01u); - code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreDwordAppliesHostOffset"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0}; - test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u}; - test.storage_buffer_range_dwords = 1; - test.storage_buffer_offsets = {12}; - test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(4, 1); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreDwordAppliesHostOffset"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0}; + test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u}; + test.storage_buffer_range_dwords = 1; + test.storage_buffer_offsets = {12}; + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(4, 1); + test.has_user_data = true; + return test; } TestCase BufferOffsetsUsePackedLaneAndStorageFallback() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - for (u32 base: {0u, 4u}) { - AppendSMovLiteral(&code, base, 0x1000u + base * 0x1000u); - AppendSMovLiteral(&code, base + 1u, 4u << 16u); - AppendSMovLiteral(&code, base + 2u, 1u); - AppendSMovLiteral(&code, base + 3u, 1u << 24u); - } - AppendVMovLiteral(&code, 0, 0x12345678u); - code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendVMovLiteral(&code, 1, 0xabcdef01u); - code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); - code.push_back(EncodeMubuf1(1, 1, 20)); - AppendEnd(&code); + std::vector code; + for (u32 base : {0u, 4u}) { + AppendSMovLiteral(&code, base, 0x1000u + base * 0x1000u); + AppendSMovLiteral(&code, base + 1u, 4u << 16u); + AppendSMovLiteral(&code, base + 2u, 1u); + AppendSMovLiteral(&code, base + 3u, 1u << 24u); + } + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendVMovLiteral(&code, 1, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); + code.push_back(EncodeMubuf1(1, 1, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferOffsetsUsePackedLaneAndStorageFallback"; - test.code = std::move(code); - test.initial = {0, 0, 0, 0}; - test.expected = {0x12345678u, 0, 0, 0xabcdef01u}; - test.storage_buffer_range_dwords = 1; - test.storage_buffer_offsets = {0, 12}; - test.force_shader_data_storage = true; - test.opcodes = {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - return test; + TestCase test; + test.name = "BufferOffsetsUsePackedLaneAndStorageFallback"; + test.code = std::move(code); + test.initial = {0, 0, 0, 0}; + test.expected = {0x12345678u, 0, 0, 0xabcdef01u}; + test.storage_buffer_range_dwords = 1; + test.storage_buffer_offsets = {0, 12}; + test.force_shader_data_storage = true; + test.opcodes = {O::SMovB32, O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + return test; } TestCase BufferLoadVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendBufferLoadOpcode(&code, 0x08, 0, 20); - AppendVMovU32(&code, 20, 2); - AppendBufferLoadOpcode(&code, 0x09, 1, 20); - AppendVMovU32(&code, 20, 0); - AppendBufferLoadOpcode(&code, 0x0a, 2, 20); - AppendVMovU32(&code, 20, 2); - AppendBufferLoadOpcode(&code, 0x0b, 3, 20); - AppendVMovU32(&code, 20, 8); - AppendBufferLoadOpcode(&code, 0x0d, 4, 20); - AppendBufferLoadOpcode(&code, 0x0f, 6, 20); - AppendBufferLoadOpcode(&code, 0x0e, 9, 20); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendBufferLoadOpcode(&code, 0x08, 0, 20); + AppendVMovU32(&code, 20, 2); + AppendBufferLoadOpcode(&code, 0x09, 1, 20); + AppendVMovU32(&code, 20, 0); + AppendBufferLoadOpcode(&code, 0x0a, 2, 20); + AppendVMovU32(&code, 20, 2); + AppendBufferLoadOpcode(&code, 0x0b, 3, 20); + AppendVMovU32(&code, 20, 8); + AppendBufferLoadOpcode(&code, 0x0d, 4, 20); + AppendBufferLoadOpcode(&code, 0x0f, 6, 20); + AppendBufferLoadOpcode(&code, 0x0e, 9, 20); - for (u32 i = 0; i < 13u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 13u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"BufferLoadVariants", - code, - {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u}, - {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u, 0x11223344u, - 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u}, - {O::BufferLoadUbyte, O::BufferLoadSbyte, O::BufferLoadUshort, O::BufferLoadSshort, - O::BufferLoadDwordx2, O::BufferLoadDwordx3, O::BufferLoadDwordx4, O::VMovB32, - O::BufferStoreDword, O::SEndpgm}}; + return {"BufferLoadVariants", + code, + {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu, + 0xddeeff00u}, + {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u, + 0x11223344u, 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u, + 0x99aabbccu, 0xddeeff00u}, + {O::BufferLoadUbyte, O::BufferLoadSbyte, O::BufferLoadUshort, + O::BufferLoadSshort, O::BufferLoadDwordx2, O::BufferLoadDwordx3, + O::BufferLoadDwordx4, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - code.push_back(EncodeMubuf0(0x0eu, 0, true, true)); - code.push_back(EncodeMubuf1(21, 0, 21)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 21 + i, 4 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMubuf0(0x0eu, 0, true, true)); + code.push_back(EncodeMubuf1(21, 0, 21)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 21 + i, 4 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress"; - test.code = std::move(code); - test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0, 0, 0, 0}; - test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, - 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - test.opcodes = {O::VMovB32, O::BufferLoadDwordx4, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(16, 2); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadDwordx4SnapshotsOverlappingAddress"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, + 0, 0, 0, 0}; + test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, + 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.opcodes = {O::VMovB32, O::BufferLoadDwordx4, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 2); + test.has_user_data = true; + return test; } TestCase BufferLoadDwordx2SnapshotsOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - code.push_back(EncodeMubuf0(0x0du, 0, true, true)); - code.push_back(EncodeMubuf1(21, 0, 21)); - for (u32 i = 0; i < 2; i++) { - AppendStoreVgpr(&code, 21 + i, 2 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMubuf0(0x0du, 0, true, true)); + code.push_back(EncodeMubuf1(21, 0, 21)); + for (u32 i = 0; i < 2; i++) { + AppendStoreVgpr(&code, 21 + i, 2 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress"; - test.code = std::move(code); - test.initial = {0x11111111u, 0x22222222u, 0, 0}; - test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u}; - test.opcodes = {O::VMovB32, O::BufferLoadDwordx2, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(8, 2); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadDwordx2SnapshotsOverlappingAddress"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0, 0}; + test.expected = {0x11111111u, 0x22222222u, 0x11111111u, 0x22222222u}; + test.opcodes = {O::VMovB32, O::BufferLoadDwordx2, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(8, 2); + test.has_user_data = true; + return test; } TestCase BufferLoadDwordx3SnapshotsOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - code.push_back(EncodeMubuf0(0x0fu, 0, true, true)); - code.push_back(EncodeMubuf1(21, 0, 21)); - for (u32 i = 0; i < 3; i++) { - AppendStoreVgpr(&code, 21 + i, 3 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMubuf0(0x0fu, 0, true, true)); + code.push_back(EncodeMubuf1(21, 0, 21)); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, 21 + i, 3 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress"; - test.code = std::move(code); - test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0}; - test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0x11111111u, 0x22222222u, 0x33333333u}; - test.opcodes = {O::VMovB32, O::BufferLoadDwordx3, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(12, 2); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadDwordx3SnapshotsOverlappingAddress"; + test.code = std::move(code); + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0, 0, 0}; + test.expected = {0x11111111u, 0x22222222u, 0x33333333u, + 0x11111111u, 0x22222222u, 0x33333333u}; + test.opcodes = {O::VMovB32, O::BufferLoadDwordx3, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(12, 2); + test.has_user_data = true; + return test; } TestCase BufferStoreVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 0, 0xaa); - AppendBufferStoreOpcode(&code, 0x18, 0, 20); - AppendVMovU32(&code, 20, 2); - AppendVMovLiteral(&code, 1, 0x0000bbccu); - AppendBufferStoreOpcode(&code, 0x1a, 1, 20); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 2, 0x11111111u); - AppendVMovLiteral(&code, 3, 0x22222222u); - AppendBufferStoreOpcode(&code, 0x1d, 2, 20); - AppendVMovU32(&code, 20, 12); - AppendVMovLiteral(&code, 4, 0x33333333u); - AppendVMovLiteral(&code, 5, 0x44444444u); - AppendVMovLiteral(&code, 6, 0x55555555u); - AppendBufferStoreOpcode(&code, 0x1f, 4, 20); - AppendVMovU32(&code, 20, 24); - AppendVMovLiteral(&code, 7, 0x66666666u); - AppendVMovLiteral(&code, 8, 0x77777777u); - AppendVMovLiteral(&code, 9, 0x88888888u); - AppendVMovLiteral(&code, 10, 0x99999999u); - AppendBufferStoreOpcode(&code, 0x1e, 7, 20); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 0, 0xaa); + AppendBufferStoreOpcode(&code, 0x18, 0, 20); + AppendVMovU32(&code, 20, 2); + AppendVMovLiteral(&code, 1, 0x0000bbccu); + AppendBufferStoreOpcode(&code, 0x1a, 1, 20); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + AppendBufferStoreOpcode(&code, 0x1d, 2, 20); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + AppendVMovLiteral(&code, 6, 0x55555555u); + AppendBufferStoreOpcode(&code, 0x1f, 4, 20); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 7, 0x66666666u); + AppendVMovLiteral(&code, 8, 0x77777777u); + AppendVMovLiteral(&code, 9, 0x88888888u); + AppendVMovLiteral(&code, 10, 0x99999999u); + AppendBufferStoreOpcode(&code, 0x1e, 7, 20); + AppendEnd(&code); - return {"BufferStoreVariants", - code, - std::vector(10, 0), - {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, - 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u}, - {O::VMovB32, O::BufferStoreByte, O::BufferStoreShort, O::BufferStoreDwordx2, - O::BufferStoreDwordx3, O::BufferStoreDwordx4, O::SEndpgm}}; + return {"BufferStoreVariants", + code, + std::vector(10, 0), + {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, + 0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u}, + {O::VMovB32, O::BufferStoreByte, O::BufferStoreShort, + O::BufferStoreDwordx2, O::BufferStoreDwordx3, O::BufferStoreDwordx4, + O::SEndpgm}}; } TestCase BufferFormatVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendBufferLoadOpcode(&code, 0x00, 0, 20); - AppendBufferLoadOpcode(&code, 0x01, 1, 20); - AppendBufferLoadOpcode(&code, 0x02, 3, 20); - AppendBufferLoadOpcode(&code, 0x03, 6, 20); - for (u32 i = 0; i < 10u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendBufferLoadOpcode(&code, 0x00, 0, 20); + AppendBufferLoadOpcode(&code, 0x01, 1, 20); + AppendBufferLoadOpcode(&code, 0x02, 3, 20); + AppendBufferLoadOpcode(&code, 0x03, 6, 20); + for (u32 i = 0; i < 10u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - TestCase load; - load.name = "BufferFormatLoadVariants"; - load.code = code; - load.initial = {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}; - load.expected = {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u, - 0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}; - load.opcodes = {O::BufferLoadFormatX, - O::BufferLoadFormatXy, - O::BufferLoadFormatXyz, - O::BufferLoadFormatXyzw, - O::VMovB32, - O::BufferStoreDword, - O::SEndpgm}; - return load; + TestCase load; + load.name = "BufferFormatLoadVariants"; + load.code = code; + load.initial = {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}; + load.expected = {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, + 0x11121314u, 0x21222324u, 0x01020304u, 0x11121314u, + 0x21222324u, 0x31323334u}; + load.opcodes = {O::BufferLoadFormatX, + O::BufferLoadFormatXy, + O::BufferLoadFormatXyz, + O::BufferLoadFormatXyzw, + O::VMovB32, + O::BufferStoreDword, + O::SEndpgm}; + return load; } TestCase BufferLoadFormatXyzwSnapshotsOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - code.push_back(EncodeMubuf0(0x03u, 0, true, true)); - code.push_back(EncodeMubuf1(21, 0, 21)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 21 + i, 4 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMubuf0(0x03u, 0, true, true)); + code.push_back(EncodeMubuf1(21, 0, 21)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 21 + i, 4 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress"; - test.code = std::move(code); - test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, 0, 0, 0, 0}; - test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, - 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::VMovB32, O::BufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData( - 16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadFormatXyzwSnapshotsOverlappingAddress"; + test.code = std::move(code); + test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, + 0, 0, 0, 0}; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, + 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatXyzw, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData( + 16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + return test; } TestCase BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 21, 0x11111111u); - AppendVMovLiteral(&code, 22, 0x22222222u); - AppendVMovLiteral(&code, 23, 0x33333333u); - AppendVMovLiteral(&code, 24, 0x44444444u); - code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); - code.push_back(EncodeMubuf0(0x03u, 0, true, true)); - code.push_back(EncodeMubuf1(21, 0, 21)); - code.push_back(EncodeSMovB32(126, InlineU32(1))); - code.push_back(EncodeSMovB32(127, InlineU32(0))); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 21 + i, 4 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 21, 0x11111111u); + AppendVMovLiteral(&code, 22, 0x22222222u); + AppendVMovLiteral(&code, 23, 0x33333333u); + AppendVMovLiteral(&code, 24, 0x44444444u); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeMubuf0(0x03u, 0, true, true)); + code.push_back(EncodeMubuf1(21, 0, 21)); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + code.push_back(EncodeSMovB32(127, InlineU32(0))); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 21 + i, 4 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress"; - test.code = std::move(code); - test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd, 0, 0, 0, 0}; - test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu, - 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - test.opcodes = {O::VMovB32, O::SMovB64, O::BufferLoadFormatXyzw, - O::SMovB32, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData( - 16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress"; + test.code = std::move(code); + test.initial = {0xaaaaaaaa, 0xbbbbbbbb, 0xcccccccc, 0xdddddddd, 0, 0, 0, 0}; + test.expected = {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu, + 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.opcodes = {O::VMovB32, O::SMovB64, O::BufferLoadFormatXyzw, + O::SMovB32, O::BufferStoreDword, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData( + 16, 2, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); + test.has_user_data = true; + return test; } TestCase BufferFormatStoreVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0x01020304u); - AppendBufferStoreOpcode(&code, 0x04, 0, 20); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 1, 0x11121314u); - AppendVMovLiteral(&code, 2, 0x21222324u); - AppendBufferStoreOpcode(&code, 0x05, 1, 20); - AppendVMovU32(&code, 20, 12); - AppendVMovLiteral(&code, 3, 0x31323334u); - AppendVMovLiteral(&code, 4, 0x41424344u); - AppendVMovLiteral(&code, 5, 0x51525354u); - AppendBufferStoreOpcode(&code, 0x06, 3, 20); - AppendVMovU32(&code, 20, 24); - AppendVMovLiteral(&code, 6, 0x61626364u); - AppendVMovLiteral(&code, 7, 0x71727374u); - AppendVMovLiteral(&code, 8, 0x81828384u); - AppendVMovLiteral(&code, 9, 0x91929394u); - AppendBufferStoreOpcode(&code, 0x07, 6, 20); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x01020304u); + AppendBufferStoreOpcode(&code, 0x04, 0, 20); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 1, 0x11121314u); + AppendVMovLiteral(&code, 2, 0x21222324u); + AppendBufferStoreOpcode(&code, 0x05, 1, 20); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 3, 0x31323334u); + AppendVMovLiteral(&code, 4, 0x41424344u); + AppendVMovLiteral(&code, 5, 0x51525354u); + AppendBufferStoreOpcode(&code, 0x06, 3, 20); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 6, 0x61626364u); + AppendVMovLiteral(&code, 7, 0x71727374u); + AppendVMovLiteral(&code, 8, 0x81828384u); + AppendVMovLiteral(&code, 9, 0x91929394u); + AppendBufferStoreOpcode(&code, 0x07, 6, 20); + AppendEnd(&code); - return {"BufferFormatStoreVariants", - code, - std::vector(10, 0), - {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, 0x51525354u, - 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, - {O::VMovB32, O::BufferStoreFormatX, O::BufferStoreFormatXy, O::BufferStoreFormatXyz, - O::BufferStoreFormatXyzw, O::SEndpgm}}; + return {"BufferFormatStoreVariants", + code, + std::vector(10, 0), + {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, + 0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, + {O::VMovB32, O::BufferStoreFormatX, O::BufferStoreFormatXy, + O::BufferStoreFormatXyz, O::BufferStoreFormatXyzw, O::SEndpgm}}; } TestCase BufferLoadFormatXResource8UintZeroExtendsByte() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMubuf0(0x00u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendStoreVgpr(&code, 0, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMubuf0(0x00u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 1); + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadFormatXResource8UintZeroExtendsByte"; - test.code = code; - test.initial = {0x11223344u, 0}; - test.expected = {0x11223344u, 0x00000044u}; - test.opcodes = {O::VMovB32, O::BufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadFormatXResource8UintZeroExtendsByte"; + test.code = code; + test.initial = {0x11223344u, 0}; + test.expected = {0x11223344u, 0x00000044u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatX, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXResource16UintWritesHalfword() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0x0000aaaau); - code.push_back(EncodeMubuf0(0x04u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x0000aaaau); + code.push_back(EncodeMubuf0(0x04u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXResource16UintWritesHalfword"; - test.code = code; - test.initial = {0x11223344u}; - test.expected = {0x1122aaaau}; - test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 4, false, 11); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXResource16UintWritesHalfword"; + test.code = code; + test.initial = {0x11223344u}; + test.expected = {0x1122aaaau}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 4, false, 11); + test.has_user_data = true; + return test; } TestCase BufferLoadFormatXyResource88UintExtractsBytes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMubuf0(0x01u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendStoreVgpr(&code, 0, 2); - AppendStoreVgpr(&code, 1, 3); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMubuf0(0x01u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 2); + AppendStoreVgpr(&code, 1, 3); + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadFormatXyResource8_8UintExtractsBytes"; - test.code = code; - test.initial = {0x0000807fu, 0x55667788u, 0, 0}; - test.expected = {0x0000807fu, 0x55667788u, 0x0000007fu, 0x00000080u}; - test.opcodes = {O::VMovB32, O::BufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferLoadFormatXyResource8_8UintExtractsBytes"; + test.code = code; + test.initial = {0x0000807fu, 0x55667788u, 0, 0}; + test.expected = {0x0000807fu, 0x55667788u, 0x0000007fu, 0x00000080u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatXy, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18); + test.has_user_data = true; + return test; } TestCase BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMubuf0(0x01u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMubuf0(0x01u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - TestCase test; - test.name = "BufferLoadFormatXyResource8_8_8_8UnormConvertsFirstTwoComponents"; - test.code = code; - test.initial = {0x44332211u, 0xdeadbeefu}; - test.expected = {0x3d888889u, 0x3e088889u}; - test.opcodes = {O::VMovB32, O::BufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 56); - test.has_user_data = true; - return test; + TestCase test; + test.name = + "BufferLoadFormatXyResource8_8_8_8UnormConvertsFirstTwoComponents"; + test.code = code; + test.initial = {0x44332211u, 0xdeadbeefu}; + test.expected = {0x3d888889u, 0x3e088889u}; + test.opcodes = {O::VMovB32, O::BufferLoadFormatXy, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 56); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXyResource88UintWritesBytes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0x000000aau); - AppendVMovLiteral(&code, 1, 0x000000bbu); - code.push_back(EncodeMubuf0(0x05u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x000000aau); + AppendVMovLiteral(&code, 1, 0x000000bbu); + code.push_back(EncodeMubuf0(0x05u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXyResource8_8UintWritesBytes"; - test.code = code; - test.initial = {0x11223344u, 0x55667788u}; - test.expected = {0x1122bbaau, 0x55667788u}; - test.opcodes = {O::VMovB32, O::BufferStoreFormatXy, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXyResource8_8UintWritesBytes"; + test.code = code; + test.initial = {0x11223344u, 0x55667788u}; + test.expected = {0x1122bbaau, 0x55667788u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXy, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 18); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXyResource32UintWritesOneDword() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0xabcdef01u); - AppendVMovLiteral(&code, 1, 0x12345678u); - code.push_back(EncodeMubuf0(0x05u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + AppendVMovLiteral(&code, 1, 0x12345678u); + code.push_back(EncodeMubuf0(0x05u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXyResource32UintWritesOneDword"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u}; - test.expected = {0xabcdef01u, 0x22222222u}; - test.opcodes = {O::VMovB32, O::BufferStoreFormatXy, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 20); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXyResource32UintWritesOneDword"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u}; + test.expected = {0xabcdef01u, 0x22222222u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXy, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 20); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXyzResource3232UintWritesTwoDwords() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0xabcdef01u); - AppendVMovLiteral(&code, 1, 0x12345678u); - AppendVMovLiteral(&code, 2, 0x0badc0deu); - code.push_back(EncodeMubuf0(0x06u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 2, 0x0badc0deu); + code.push_back(EncodeMubuf0(0x06u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXyzResource32_32UintWritesTwoDwords"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u, 0x33333333u}; - test.expected = {0xabcdef01u, 0x12345678u, 0x33333333u}; - test.opcodes = {O::VMovB32, O::BufferStoreFormatXyz, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 62); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXyzResource32_32UintWritesTwoDwords"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u}; + test.expected = {0xabcdef01u, 0x12345678u, 0x33333333u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXyz, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 62); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXyzwResource323232UintWritesThreeDwords() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0xabcdef01u); - AppendVMovLiteral(&code, 1, 0x12345678u); - AppendVMovLiteral(&code, 2, 0x0badc0deu); - AppendVMovLiteral(&code, 3, 0xfeedfaceu); - code.push_back(EncodeMubuf0(0x07u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + AppendVMovLiteral(&code, 1, 0x12345678u); + AppendVMovLiteral(&code, 2, 0x0badc0deu); + AppendVMovLiteral(&code, 3, 0xfeedfaceu); + code.push_back(EncodeMubuf0(0x07u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXyzwResource32_32_32UintWritesThreeDwords"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - test.expected = {0xabcdef01u, 0x12345678u, 0x0badc0deu, 0x44444444u}; - test.opcodes = {O::VMovB32, O::BufferStoreFormatXyzw, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 72); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXyzwResource32_32_32UintWritesThreeDwords"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + test.expected = {0xabcdef01u, 0x12345678u, 0x0badc0deu, 0x44444444u}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXyzw, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 72); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXyzResource8UintWritesOneByte() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0x000000aau); - AppendVMovLiteral(&code, 1, 0x000000bbu); - AppendVMovLiteral(&code, 2, 0x000000ccu); - code.push_back(EncodeMubuf0(0x06u)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x000000aau); + AppendVMovLiteral(&code, 1, 0x000000bbu); + AppendVMovLiteral(&code, 2, 0x000000ccu); + code.push_back(EncodeMubuf0(0x06u)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXyzResource8UintWritesOneByte"; - test.code = code; - test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu}; - test.expected = {0x112233aau, 0x55667788u, 0x99aabbccu}; - test.opcodes = {O::VMovB32, O::BufferStoreFormatXyz, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXyzResource8UintWritesOneByte"; + test.code = code; + test.initial = {0x11223344u, 0x55667788u, 0x99aabbccu}; + test.expected = {0x112233aau, 0x55667788u, 0x99aabbccu}; + test.opcodes = {O::VMovB32, O::BufferStoreFormatXyz, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(0, 8, false, 5); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXAddTidUsesLaneIndex() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x12345678u); - code.push_back(EncodeMubuf0(0x04u, 0, false, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeMubuf0(0x04u, 0, false, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXAddTidUsesLaneIndex"; - test.code = code; - test.initial = std::vector(4, 0); - test.expected = std::vector(4, 0x12345678u); - test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - test.user_data = MakeStructuredStorageBufferData(4, 4, true); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXAddTidUsesLaneIndex"; + test.code = code; + test.initial = std::vector(4, 0); + test.expected = std::vector(4, 0x12345678u); + test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + test.user_data = MakeStructuredStorageBufferData(4, 4, true); + test.has_user_data = true; + return test; } TestCase BufferStoreFormatXDropsOutOfRangeRecord() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovLiteral(&code, 0, 0xabcdef01u); - code.push_back(EncodeMubuf0(0x04u, 0, true, false)); - code.push_back(EncodeMubuf1(0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovLiteral(&code, 0, 0xabcdef01u); + code.push_back(EncodeMubuf0(0x04u, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "BufferStoreFormatXDropsOutOfRangeRecord"; - test.code = code; - test.initial = {0x11111111u, 0x22222222u}; - test.expected = {0x11111111u, 0x22222222u}; - test.storage_buffer_range_dwords = 1; - test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20); - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferStoreFormatXDropsOutOfRangeRecord"; + test.code = code; + test.initial = {0x11111111u, 0x22222222u}; + test.expected = {0x11111111u, 0x22222222u}; + test.storage_buffer_range_dwords = 1; + test.opcodes = {O::VMovB32, O::BufferStoreFormatX, O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20); + test.has_user_data = true; + return test; } TestCase TBufferLoadVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadOpcode(&code, 0x00, 0, 20); - AppendTBufferLoadOpcode(&code, 0x01, 1, 20); - AppendTBufferLoadOpcode(&code, 0x02, 3, 20); - AppendTBufferLoadOpcode(&code, 0x03, 6, 20); - for (u32 i = 0; i < 10u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadOpcode(&code, 0x00, 0, 20); + AppendTBufferLoadOpcode(&code, 0x01, 1, 20); + AppendTBufferLoadOpcode(&code, 0x02, 3, 20); + AppendTBufferLoadOpcode(&code, 0x03, 6, 20); + for (u32 i = 0; i < 10u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadVariants", - code, - {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, - {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u, 0x21222324u, - 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, - {O::TBufferLoadFormatX, O::TBufferLoadFormatXy, O::TBufferLoadFormatXyz, - O::TBufferLoadFormatXyzw, O::VMovB32, O::BufferStoreDword, O::SEndpgm}}; + return {"TBufferLoadVariants", + code, + {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, + {0x01020304u, 0x01020304u, 0x11121314u, 0x01020304u, 0x11121314u, + 0x21222324u, 0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u}, + {O::TBufferLoadFormatX, O::TBufferLoadFormatXy, + O::TBufferLoadFormatXyz, O::TBufferLoadFormatXyzw, O::VMovB32, + O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXyzwSnapshotsOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - constexpr auto format = BufferFormat(Prospero::BufferFormat::k32_32_32_32Float); - code.push_back(EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true)); - code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 21 + i, 4 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + constexpr auto format = + BufferFormat(Prospero::BufferFormat::k32_32_32_32Float); + code.push_back( + EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true)); + code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 21 + i, 4 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress"; - test.code = std::move(code); - test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, 0, 0, 0, 0}; - test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, - 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(16, 2); - test.has_user_data = true; - return test; + TestCase test; + test.name = "TBufferLoadFormatXyzwSnapshotsOverlappingAddress"; + test.code = std::move(code); + test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, + 0, 0, 0, 0}; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u, + 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 2); + test.has_user_data = true; + return test; } TestCase TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt); - code.push_back(EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true)); - code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, 21 + i, 4 + i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + constexpr auto format = BufferFormat(Prospero::BufferFormat::k8_8_8_8UInt); + code.push_back( + EncodeMtbuf0(0x03u, format & 0xfu, (format >> 4u) & 0x7u, 0, true, true)); + code.push_back(EncodeMtbuf1(0x03u, 21, 0, 21)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, 21 + i, 4 + i); + } + AppendEnd(&code); - TestCase test; - test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress"; - test.code = std::move(code); - test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0}; - test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u}; - test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(4, 8); - test.has_user_data = true; - return test; + TestCase test; + test.name = "TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress"; + test.code = std::move(code); + test.initial = {0x44332211u, 0, 0, 0, 0, 0, 0, 0}; + test.expected = {0x44332211u, 0, 0, 0, 0x11u, 0x22u, 0x33u, 0x44u}; + test.opcodes = {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(4, 8); + test.has_user_data = true; + return test; } TestCase TBufferStoreFormatX8UintWritesOneByte() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 0, 0xaa); - code.push_back(EncodeMtbuf0(0x04, 5, 0)); - code.push_back(EncodeMtbuf1(0x04, 0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 0xaa); + code.push_back(EncodeMtbuf0(0x04, 5, 0)); + code.push_back(EncodeMtbuf1(0x04, 0, 0, 20)); + AppendEnd(&code); - return {"TBufferStoreFormatX8UintWritesOneByte", - code, - {0x11223344u}, - {0x112233aau}, - {O::VMovB32, O::TBufferStoreFormatX, O::SEndpgm}}; + return {"TBufferStoreFormatX8UintWritesOneByte", + code, + {0x11223344u}, + {0x112233aau}, + {O::VMovB32, O::TBufferStoreFormatX, O::SEndpgm}}; } TestCase TBufferLoadFormatX8UintZeroExtendsByte() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x00, 5, 0)); - code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x00, 5, 0)); + code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - return {"TBufferLoadFormatX8UintZeroExtendsByte", - code, - {0x11223344u, 0}, - {0x00000044u, 0}, - {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; + return {"TBufferLoadFormatX8UintZeroExtendsByte", + code, + {0x11223344u, 0}, + {0x00000044u, 0}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatX8888UintExtractsFirstByte() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20, Prospero::BufferFormat::k8_8_8_8UInt); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20, + Prospero::BufferFormat::k8_8_8_8UInt); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - return {"TBufferLoadFormatX8_8_8_8UintExtractsFirstByte", - code, - {0x44332211u, 0}, - {0x00000011u, 0}, - {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; + return {"TBufferLoadFormatX8_8_8_8UintExtractsFirstByte", + code, + {0x44332211u, 0}, + {0x00000011u, 0}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXIdxenUsesDescriptorStride() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - code.push_back(EncodeMtbuf0(0x00, 5, 0, 0, true, false)); - code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + code.push_back(EncodeMtbuf0(0x00, 5, 0, 0, true, false)); + code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "TBufferLoadFormatXIdxenUsesDescriptorStride"; - test.code = code; - test.initial = {0x01020304u, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu}; - test.expected = {0x0000007eu, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu}; - test.opcodes = {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}; - test.user_data = MakeStructuredStorageBufferData(16, 3); - test.has_user_data = true; - return test; + TestCase test; + test.name = "TBufferLoadFormatXIdxenUsesDescriptorStride"; + test.code = code; + test.initial = {0x01020304u, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu}; + test.expected = {0x0000007eu, 0, 0, 0, 0, 0, 0, 0, 0x0000007eu}; + test.opcodes = {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, + O::SEndpgm}; + test.user_data = MakeStructuredStorageBufferData(16, 3); + test.has_user_data = true; + return test; } TestCase TBufferLoadFormatX16FloatConvertsToFloat() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20, Prospero::BufferFormat::k16Float); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x00, 0, 20, + Prospero::BufferFormat::k16Float); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - return {"TBufferLoadFormatX16FloatConvertsToFloat", - code, - {0x00003c00u, 0}, - {0x3f800000u, 0}, - {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; + return {"TBufferLoadFormatX16FloatConvertsToFloat", + code, + {0x00003c00u, 0}, + {0x3f800000u, 0}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXSintSignExtendsSubDword() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x00, 6, 0)); - code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); - AppendVMovU32(&code, 20, 4); - code.push_back(EncodeMtbuf0(0x00, 12, 0)); - code.push_back(EncodeMtbuf1(0x00, 1, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x00, 6, 0)); + code.push_back(EncodeMtbuf1(0x00, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + code.push_back(EncodeMtbuf0(0x00, 12, 0)); + code.push_back(EncodeMtbuf1(0x00, 1, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - return {"TBufferLoadFormatXSintSignExtendsSubDword", - code, - {0x00000080u, 0x00008001u}, - {0xffffff80u, 0xffff8001u}, - {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; + return {"TBufferLoadFormatXSintSignExtendsSubDword", + code, + {0x00000080u, 0x00008001u}, + {0xffffff80u, 0xffff8001u}, + {O::VMovB32, O::TBufferLoadFormatX, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferStoreFormatXSintWritesSubDword() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0xffffff80u); - code.push_back(EncodeMtbuf0(0x04, 6, 0)); - code.push_back(EncodeMtbuf1(0x04, 0, 0, 20)); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 1, 0xffff8001u); - code.push_back(EncodeMtbuf0(0x04, 12, 0)); - code.push_back(EncodeMtbuf1(0x04, 1, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0xffffff80u); + code.push_back(EncodeMtbuf0(0x04, 6, 0)); + code.push_back(EncodeMtbuf1(0x04, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 1, 0xffff8001u); + code.push_back(EncodeMtbuf0(0x04, 12, 0)); + code.push_back(EncodeMtbuf1(0x04, 1, 0, 20)); + AppendEnd(&code); - return {"TBufferStoreFormatXSintWritesSubDword", - code, - {0x11223344u, 0x55667788u}, - {0x11223380u, 0x55668001u}, - {O::VMovB32, O::TBufferStoreFormatX, O::SEndpgm}}; + return {"TBufferStoreFormatXSintWritesSubDword", + code, + {0x11223344u, 0x55667788u}, + {0x11223380u, 0x55668001u}, + {O::VMovB32, O::TBufferStoreFormatX, O::SEndpgm}}; } TestCase TBufferLoadFormatXy88IntegerComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x01, 2, 1)); - code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); - AppendVMovU32(&code, 20, 4); - code.push_back(EncodeMtbuf0(0x01, 3, 1)); - code.push_back(EncodeMtbuf1(0x01, 2, 0, 20)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 2, 1)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + code.push_back(EncodeMtbuf0(0x01, 3, 1)); + code.push_back(EncodeMtbuf1(0x01, 2, 0, 20)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXy8_8IntegerComponents", - code, - {0x0000807fu, 0x00007f80u, 0, 0}, - {0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu}, - {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXy8_8IntegerComponents", + code, + {0x0000807fu, 0x00007f80u, 0, 0}, + {0x0000007fu, 0x00000080u, 0xffffff80u, 0x0000007fu}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferStoreFormatXy88IntegerComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 0, 0xaa); - AppendVMovU32(&code, 1, 0xbb); - code.push_back(EncodeMtbuf0(0x05, 2, 1)); - code.push_back(EncodeMtbuf1(0x05, 0, 0, 20)); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 2, 0xffffff80u); - AppendVMovU32(&code, 3, 0x7f); - code.push_back(EncodeMtbuf0(0x05, 3, 1)); - code.push_back(EncodeMtbuf1(0x05, 2, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 0xaa); + AppendVMovU32(&code, 1, 0xbb); + code.push_back(EncodeMtbuf0(0x05, 2, 1)); + code.push_back(EncodeMtbuf1(0x05, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0xffffff80u); + AppendVMovU32(&code, 3, 0x7f); + code.push_back(EncodeMtbuf0(0x05, 3, 1)); + code.push_back(EncodeMtbuf1(0x05, 2, 0, 20)); + AppendEnd(&code); - return {"TBufferStoreFormatXy8_8IntegerComponents", - code, - {0x11223344u, 0x55667788u}, - {0x1122bbaau, 0x55667f80u}, - {O::VMovB32, O::TBufferStoreFormatXy, O::SEndpgm}}; + return {"TBufferStoreFormatXy8_8IntegerComponents", + code, + {0x11223344u, 0x55667788u}, + {0x1122bbaau, 0x55667f80u}, + {O::VMovB32, O::TBufferStoreFormatXy, O::SEndpgm}}; } TestCase TBufferLoadFormatXy1616IntegerComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x01, 11, 1)); - code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); - AppendVMovU32(&code, 20, 4); - code.push_back(EncodeMtbuf0(0x01, 12, 1)); - code.push_back(EncodeMtbuf1(0x01, 2, 0, 20)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 11, 1)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + code.push_back(EncodeMtbuf0(0x01, 12, 1)); + code.push_back(EncodeMtbuf1(0x01, 2, 0, 20)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXy16_16IntegerComponents", - code, - {0x80017fffu, 0x7fff8000u, 0, 0}, - {0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu}, - {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXy16_16IntegerComponents", + code, + {0x80017fffu, 0x7fff8000u, 0, 0}, + {0x00007fffu, 0x00008001u, 0xffff8000u, 0x00007fffu}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferStoreFormatXy1616IntegerComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 0, 0xaaaa); - AppendVMovU32(&code, 1, 0xbbbb); - code.push_back(EncodeMtbuf0(0x05, 11, 1)); - code.push_back(EncodeMtbuf1(0x05, 0, 0, 20)); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 2, 0xffff8000u); - AppendVMovU32(&code, 3, 0x7fff); - code.push_back(EncodeMtbuf0(0x05, 12, 1)); - code.push_back(EncodeMtbuf1(0x05, 2, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 0xaaaa); + AppendVMovU32(&code, 1, 0xbbbb); + code.push_back(EncodeMtbuf0(0x05, 11, 1)); + code.push_back(EncodeMtbuf1(0x05, 0, 0, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0xffff8000u); + AppendVMovU32(&code, 3, 0x7fff); + code.push_back(EncodeMtbuf0(0x05, 12, 1)); + code.push_back(EncodeMtbuf1(0x05, 2, 0, 20)); + AppendEnd(&code); - return {"TBufferStoreFormatXy16_16IntegerComponents", - code, - {0x11223344u, 0x55667788u}, - {0xbbbbaaaau, 0x7fff8000u}, - {O::VMovB32, O::TBufferStoreFormatXy, O::SEndpgm}}; + return {"TBufferStoreFormatXy16_16IntegerComponents", + code, + {0x11223344u, 0x55667788u}, + {0xbbbbaaaau, 0x7fff8000u}, + {O::VMovB32, O::TBufferStoreFormatXy, O::SEndpgm}}; } TestCase TBufferLoadFormatXyz16161616UintLoadsHalfwords() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20, Prospero::BufferFormat::k16_16_16_16UInt); - for (u32 i = 0; i < 3; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20, + Prospero::BufferFormat::k16_16_16_16UInt); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords", - code, - {0x22221111u, 0x44443333u, 0}, - {0x00001111u, 0x00002222u, 0x00003333u}, - {O::VMovB32, O::TBufferLoadFormatXyz, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXyz16_16_16_16UintLoadsHalfwords", + code, + {0x22221111u, 0x44443333u, 0}, + {0x00001111u, 0x00002222u, 0x00003333u}, + {O::VMovB32, O::TBufferLoadFormatXyz, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXy88UnormConvertsToFloat() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x01, 14, 0)); - code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 14, 0)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - return {"TBufferLoadFormatXy8_8UnormConvertsToFloat", - code, - {0x0000ff80u, 0}, - {0x3f008081u, 0x3f800000u}, - {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXy8_8UnormConvertsToFloat", + code, + {0x0000ff80u, 0}, + {0x3f008081u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXy88SnormConvertsToFloat() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x01, 0, 20, Prospero::BufferFormat::k8_8SNorm); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x01, 0, 20, + Prospero::BufferFormat::k8_8SNorm); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - return {"TBufferLoadFormatXy8_8SnormConvertsToFloat", - code, - {0x00007f80u, 0}, - {0xbf800000u, 0x3f800000u}, - {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXy8_8SnormConvertsToFloat", + code, + {0x00007f80u, 0}, + {0xbf800000u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXy1616UnormConvertsToFloat() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x01, 7, 1)); - code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 7, 1)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - return {"TBufferLoadFormatXy16_16UnormConvertsToFloat", - code, - {0xffff8000u, 0}, - {0x3f000080u, 0x3f800000u}, - {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXy16_16UnormConvertsToFloat", + code, + {0xffff8000u, 0}, + {0x3f000080u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x01, 8, 3)); - code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x01, 8, 3)); + code.push_back(EncodeMtbuf1(0x01, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - return {"TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents", - code, - {0x44332211u, 0xdeadbeefu}, - {0x3d888889u, 0x3e088889u}, - {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXy8_8_8_8UnormConvertsFirstTwoComponents", + code, + {0x44332211u, 0xdeadbeefu}, + {0x3d888889u, 0x3e088889u}, + {O::VMovB32, O::TBufferLoadFormatXy, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXyzw8888UintExtractsBytes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20, Prospero::BufferFormat::k8_8_8_8UInt); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20, + Prospero::BufferFormat::k8_8_8_8UInt); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes", - code, - {0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu}, - {0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u}, - {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXyzw8_8_8_8UintExtractsBytes", + code, + {0x44332211u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu}, + {0x00000011u, 0x00000022u, 0x00000033u, 0x00000044u}, + {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXyzw1010102SnormConvertsToFloat() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20, Prospero::BufferFormat::k10_10_10_2SNorm); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x03, 0, 20, + Prospero::BufferFormat::k10_10_10_2SNorm); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat", - code, - {0x800801ffu, 0, 0, 0}, - {0x3f800000u, 0xbf800000u, 0, 0xbf800000u}, - {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXyzw10_10_10_2SnormConvertsToFloat", + code, + {0x800801ffu, 0, 0, 0}, + {0x3f800000u, 0xbf800000u, 0, 0xbf800000u}, + {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXyz111110FloatUnpacks() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20, Prospero::BufferFormat::k11_11_10Float); - for (u32 i = 0; i < 3; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendTBufferLoadFormatOpcode(&code, 0x02, 0, 20, + Prospero::BufferFormat::k11_11_10Float); + for (u32 i = 0; i < 3; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXyz11_11_10FloatUnpacks", - code, - {0x781e03c0u, 0, 0}, - {0x3f800000u, 0x3f800000u, 0x3f800000u}, - {O::VMovB32, O::TBufferLoadFormatXyz, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXyz11_11_10FloatUnpacks", + code, + {0x781e03c0u, 0, 0}, + {0x3f800000u, 0x3f800000u, 0x3f800000u}, + {O::VMovB32, O::TBufferLoadFormatXyz, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferLoadFormatXyzw3232FloatZerosMissingComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeMtbuf0(0x03, 0, 4)); - code.push_back(EncodeMtbuf1(0x03, 0, 0, 20)); - for (u32 i = 0; i < 4; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeMtbuf0(0x03, 0, 4)); + code.push_back(EncodeMtbuf1(0x03, 0, 0, 20)); + for (u32 i = 0; i < 4; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - return {"TBufferLoadFormatXyzw32_32FloatZerosMissingComponents", - code, - {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, - {0x11111111u, 0x22222222u, 0, 0}, - {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; + return { + "TBufferLoadFormatXyzw32_32FloatZerosMissingComponents", + code, + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, + {0x11111111u, 0x22222222u, 0, 0}, + {O::VMovB32, O::TBufferLoadFormatXyzw, O::BufferStoreDword, O::SEndpgm}}; } TestCase TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x40000000u); - AppendVMovLiteral(&code, 2, 0x40400000u); - AppendVMovLiteral(&code, 3, 0x40800000u); - code.push_back(EncodeMtbuf0(0x07, 0, 4)); - code.push_back(EncodeMtbuf1(0x07, 0, 0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40400000u); + AppendVMovLiteral(&code, 3, 0x40800000u); + code.push_back(EncodeMtbuf0(0x07, 0, 4)); + code.push_back(EncodeMtbuf1(0x07, 0, 0, 20)); + AppendEnd(&code); - return {"TBufferStoreFormatXyzw32_32FloatWritesOnlyPresentComponents", - code, - {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu}, - {0x3f800000u, 0x40000000u, 0xccccccccu, 0xddddddddu}, - {O::VMovB32, O::TBufferStoreFormatXyzw, O::SEndpgm}}; + return {"TBufferStoreFormatXyzw32_32FloatWritesOnlyPresentComponents", + code, + {0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0xddddddddu}, + {0x3f800000u, 0x40000000u, 0xccccccccu, 0xddddddddu}, + {O::VMovB32, O::TBufferStoreFormatXyzw, O::SEndpgm}}; } TestCase TBufferStoreVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 0, 0x01020304u); - AppendTBufferStoreOpcode(&code, 0x04, 0, 20); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 1, 0x11121314u); - AppendVMovLiteral(&code, 2, 0x21222324u); - AppendTBufferStoreOpcode(&code, 0x05, 1, 20); - AppendVMovU32(&code, 20, 12); - AppendVMovLiteral(&code, 3, 0x31323334u); - AppendVMovLiteral(&code, 4, 0x41424344u); - AppendVMovLiteral(&code, 5, 0x51525354u); - AppendTBufferStoreOpcode(&code, 0x06, 3, 20); - AppendVMovU32(&code, 20, 24); - AppendVMovLiteral(&code, 6, 0x61626364u); - AppendVMovLiteral(&code, 7, 0x71727374u); - AppendVMovLiteral(&code, 8, 0x81828384u); - AppendVMovLiteral(&code, 9, 0x91929394u); - AppendTBufferStoreOpcode(&code, 0x07, 6, 20); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 0, 0x01020304u); + AppendTBufferStoreOpcode(&code, 0x04, 0, 20); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 1, 0x11121314u); + AppendVMovLiteral(&code, 2, 0x21222324u); + AppendTBufferStoreOpcode(&code, 0x05, 1, 20); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 3, 0x31323334u); + AppendVMovLiteral(&code, 4, 0x41424344u); + AppendVMovLiteral(&code, 5, 0x51525354u); + AppendTBufferStoreOpcode(&code, 0x06, 3, 20); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 6, 0x61626364u); + AppendVMovLiteral(&code, 7, 0x71727374u); + AppendVMovLiteral(&code, 8, 0x81828384u); + AppendVMovLiteral(&code, 9, 0x91929394u); + AppendTBufferStoreOpcode(&code, 0x07, 6, 20); + AppendEnd(&code); - return {"TBufferStoreVariants", - code, - std::vector(10, 0), - {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, 0x51525354u, - 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, - {O::VMovB32, O::TBufferStoreFormatX, O::TBufferStoreFormatXy, O::TBufferStoreFormatXyz, - O::TBufferStoreFormatXyzw, O::SEndpgm}}; + return {"TBufferStoreVariants", + code, + std::vector(10, 0), + {0x01020304u, 0x11121314u, 0x21222324u, 0x31323334u, 0x41424344u, + 0x51525354u, 0x61626364u, 0x71727374u, 0x81828384u, 0x91929394u}, + {O::VMovB32, O::TBufferStoreFormatX, O::TBufferStoreFormatXy, + O::TBufferStoreFormatXyz, O::TBufferStoreFormatXyzw, O::SEndpgm}}; } TestCase FlatLoadVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeFlat0(0x08, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x09, 0, 2)); - code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0a, 0, 0)); - code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0b, 0, 2)); - code.push_back(EncodeFlat1(3, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0c, 0, 8)); - code.push_back(EncodeFlat1(13, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0d, 0, 8)); - code.push_back(EncodeFlat1(4, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0f, 0, 8)); - code.push_back(EncodeFlat1(6, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0e, 0, 8)); - code.push_back(EncodeFlat1(9, 0x7d, 0, 20)); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 0); + code.push_back(EncodeFlat0(0x08, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x09, 0, 2)); + code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0a, 0, 0)); + code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0b, 0, 2)); + code.push_back(EncodeFlat1(3, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0c, 0, 8)); + code.push_back(EncodeFlat1(13, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0d, 0, 8)); + code.push_back(EncodeFlat1(4, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0f, 0, 8)); + code.push_back(EncodeFlat1(6, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0e, 0, 8)); + code.push_back(EncodeFlat1(9, 0x7d, 0, 20)); - for (u32 i = 0; i < 14u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + for (u32 i = 0; i < 14u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - TestCase test {"FlatLoadVariants", - code, - {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u}, - {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, 0x55667788u, 0x11223344u, - 0x55667788u, 0x99aabbccu, 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u, - 0x11223344u}, - {O::VMovB32, O::FlatLoadUbyte, O::FlatLoadSbyte, O::FlatLoadUshort, - O::FlatLoadSshort, O::FlatLoadDword, O::FlatLoadDwordx2, O::FlatLoadDwordx3, - O::FlatLoadDwordx4, O::BufferStoreDword, O::SEndpgm}}; - test.flat_memory_base = 0; - return test; + TestCase test{"FlatLoadVariants", + code, + {0x80ff7f01u, 0x00008001u, 0x11223344u, 0x55667788u, + 0x99aabbccu, 0xddeeff00u}, + {0x01u, 0xffffffffu, 0x7f01u, 0xffff80ffu, 0x11223344u, + 0x55667788u, 0x11223344u, 0x55667788u, 0x99aabbccu, + 0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u, + 0x11223344u}, + {O::VMovB32, O::FlatLoadUbyte, O::FlatLoadSbyte, + O::FlatLoadUshort, O::FlatLoadSshort, O::FlatLoadDword, + O::FlatLoadDwordx2, O::FlatLoadDwordx3, O::FlatLoadDwordx4, + O::BufferStoreDword, O::SEndpgm}}; + test.flat_memory_base = 0; + return test; } TestCase BranchVccnzUsesCarryProducedWholeMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVop2(0x1a, 3, InlineU32(2), 1), - EncodeVopc(0xc0, Vgpr(0), 0), - }; - AppendVMovLiteral(&code, 2, 0xffffffffu); - code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 1)); - code.push_back(EncodeSopp(0x07, 2)); - code.push_back(EncodeVop1(0x01, 2, InlineU32(11))); - code.push_back(EncodeSopp(0x02, 1)); - AppendVMovU32(&code, 2, 42); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), EncodeVop2(0x1a, 3, InlineU32(2), 1), + EncodeVopc(0xc0, Vgpr(0), 0), + }; + AppendVMovLiteral(&code, 2, 0xffffffffu); + code.push_back(EncodeVop2(0x28, 5, Vgpr(2), 1)); + code.push_back(EncodeSopp(0x07, 2)); + code.push_back(EncodeVop1(0x01, 2, InlineU32(11))); + code.push_back(EncodeSopp(0x02, 1)); + AppendVMovU32(&code, 2, 42); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "BranchVccnzUsesCarryProducedWholeMask"; - test.code = code; - test.expected = std::vector(8, 42); - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::VCmpFU32, O::VAddcU32, - O::SCbranchVccnz, O::SBranch, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "BranchVccnzUsesCarryProducedWholeMask"; + test.code = code; + test.expected = std::vector(8, 42); + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, + O::VCmpFU32, O::VAddcU32, O::SCbranchVccnz, + O::SBranch, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase ScalarLoadAlignsComponentsAndMasksAddress() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 0, 1u); - AppendSMovLiteral(&code, 2, 3u); - AppendSMovLiteral(&code, 3, 0xffff0000u); - code.push_back(EncodeSmem0(0x00, 1, 1)); - code.push_back(EncodeSmem1(3, 0)); - AppendStoreSgpr(&code, 1, 0); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 0, 1u); + AppendSMovLiteral(&code, 2, 3u); + AppendSMovLiteral(&code, 3, 0xffff0000u); + code.push_back(EncodeSmem0(0x00, 1, 1)); + code.push_back(EncodeSmem1(3, 0)); + AppendStoreSgpr(&code, 1, 0); + AppendEnd(&code); - return {"ScalarLoadAlignsComponentsAndMasksAddress", - code, - {0x11111111u, 0x22222222u}, - {0x11111111u, 0x22222222u}, - {O::SMovB32, O::SLoadDword, O::BufferStoreDword, O::SEndpgm}}; + return {"ScalarLoadAlignsComponentsAndMasksAddress", + code, + {0x11111111u, 0x22222222u}, + {0x11111111u, 0x22222222u}, + {O::SMovB32, O::SLoadDword, O::BufferStoreDword, O::SEndpgm}}; } TestCase FlatVirtualAddressRebasesGuestAllocation() { - using O = ShaderOpcode; + using O = ShaderOpcode; - constexpr uint64_t GuestBase = 0x0000000110000000ull; - std::vector code; - AppendVMovLiteral(&code, 20, static_cast(GuestBase + 4u)); - AppendVMovLiteral(&code, 21, static_cast(GuestBase >> 32u)); - code.push_back(EncodeFlat0(0x0c, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); - code.push_back(EncodeFlat0(0x0c, 2, 0)); - code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); - AppendVMovLiteral(&code, 21, static_cast((GuestBase >> 32u) + 1u)); - code.push_back(EncodeFlat0(0x0c, 0, 0)); - code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendStoreVgpr(&code, 2, 2); - AppendEnd(&code); + constexpr uint64_t GuestBase = 0x0000000110000000ull; + std::vector code; + AppendVMovLiteral(&code, 20, static_cast(GuestBase + 4u)); + AppendVMovLiteral(&code, 21, static_cast(GuestBase >> 32u)); + code.push_back(EncodeFlat0(0x0c, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); + code.push_back(EncodeFlat0(0x0c, 2, 0)); + code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); + AppendVMovLiteral(&code, 21, static_cast((GuestBase >> 32u) + 1u)); + code.push_back(EncodeFlat0(0x0c, 0, 0)); + code.push_back(EncodeFlat1(2, 0x7d, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendStoreVgpr(&code, 2, 2); + AppendEnd(&code); - TestCase test; - test.name = "FlatVirtualAddressRebasesGuestAllocation"; - test.code = std::move(code); - test.initial = {0, 0x12345678u}; - test.expected = {0x12345678u, 0x12345678u, 0}; - test.flat_memory_base = GuestBase; - test.opcodes = {O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, O::SEndpgm}; - return test; + TestCase test; + test.name = "FlatVirtualAddressRebasesGuestAllocation"; + test.code = std::move(code); + test.initial = {0, 0x12345678u}; + test.expected = {0x12345678u, 0x12345678u, 0}; + test.flat_memory_base = GuestBase; + test.opcodes = {O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, + O::SEndpgm}; + return test; } TestCase GlobalSignedImmediateRebasesBeforeSaddr() { - using O = ShaderOpcode; + using O = ShaderOpcode; - constexpr uint64_t GuestBase = 0x0000000110000000ull; - std::vector code; - AppendSMovLiteral(&code, 0, static_cast(GuestBase)); - AppendSMovLiteral(&code, 1, static_cast(GuestBase >> 32u)); - AppendVMovU32(&code, 20, 8); - code.push_back(EncodeFlat0(0x0c, 2, 0xffcu)); - code.push_back(EncodeFlat1(0, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + constexpr uint64_t GuestBase = 0x0000000110000000ull; + std::vector code; + AppendSMovLiteral(&code, 0, static_cast(GuestBase)); + AppendSMovLiteral(&code, 1, static_cast(GuestBase >> 32u)); + AppendVMovU32(&code, 20, 8); + code.push_back(EncodeFlat0(0x0c, 2, 0xffcu)); + code.push_back(EncodeFlat1(0, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - return {"GlobalSignedImmediateRebasesBeforeSaddr", - code, - {0x11111111u, 0x22222222u, 0x12345678u}, - {0x12345678u}, - {O::SMovB32, O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, O::SEndpgm}}; + return {"GlobalSignedImmediateRebasesBeforeSaddr", + code, + {0x11111111u, 0x22222222u, 0x12345678u}, + {0x12345678u}, + {O::SMovB32, O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, + O::SEndpgm}}; } TestCase FlatSegmentIgnoresSaddrAndMasksOffsetMsb() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(4))); - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeFlat0(0x0c, 0, 4)); - code.push_back(EncodeFlat1(0, 0, 0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeFlat0(0x0c, 0, 0x804)); - code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(4))); + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 0); + code.push_back(EncodeFlat0(0x0c, 0, 4)); + code.push_back(EncodeFlat1(0, 0, 0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeFlat0(0x0c, 0, 0x804)); + code.push_back(EncodeFlat1(1, 0x7d, 0, 20)); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - std::vector initial(514, 0); - initial[1] = 0x11111111u; - initial[2] = 0x22222222u; - initial[513] = 0x33333333u; + std::vector initial(514, 0); + initial[1] = 0x11111111u; + initial[2] = 0x22222222u; + initial[513] = 0x33333333u; - TestCase test; - test.name = "FlatSegmentIgnoresSaddrAndMasksOffsetMsb"; - test.code = code; - test.initial = initial; - test.expected = {0x11111111u, 0x11111111u}; - test.opcodes = {O::SMovB32, O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, O::SEndpgm}; - test.flat_memory_base = 0; - return test; + TestCase test; + test.name = "FlatSegmentIgnoresSaddrAndMasksOffsetMsb"; + test.code = code; + test.initial = initial; + test.expected = {0x11111111u, 0x11111111u}; + test.opcodes = {O::SMovB32, O::VMovB32, O::FlatLoadDword, O::BufferStoreDword, + O::SEndpgm}; + test.flat_memory_base = 0; + return test; } TestCase FlatStoreVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 0, 0xaa); - code.push_back(EncodeFlat0(0x18, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); - AppendVMovU32(&code, 20, 2); - AppendVMovLiteral(&code, 1, 0x0000bbccu); - code.push_back(EncodeFlat0(0x1a, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 1, 20)); - AppendVMovU32(&code, 20, 4); - AppendVMovLiteral(&code, 2, 0x11111111u); - AppendVMovLiteral(&code, 3, 0x22222222u); - code.push_back(EncodeFlat0(0x1d, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 2, 20)); - AppendVMovU32(&code, 20, 12); - AppendVMovLiteral(&code, 4, 0x33333333u); - AppendVMovLiteral(&code, 5, 0x44444444u); - AppendVMovLiteral(&code, 6, 0x55555555u); - code.push_back(EncodeFlat0(0x1f, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 4, 20)); - AppendVMovU32(&code, 20, 24); - AppendVMovLiteral(&code, 7, 0x66666666u); - AppendVMovLiteral(&code, 8, 0x77777777u); - AppendVMovLiteral(&code, 9, 0x88888888u); - AppendVMovLiteral(&code, 10, 0x99999999u); - code.push_back(EncodeFlat0(0x1e, 0, 0)); - code.push_back(EncodeFlat1(0, 0x7d, 7, 20)); - AppendVMovU32(&code, 20, 0); - code.push_back(EncodeFlat0(0x1c, 0, 40)); - code.push_back(EncodeFlat1(0, 0x7d, 2, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 0, 0xaa); + code.push_back(EncodeFlat0(0x18, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 0, 20)); + AppendVMovU32(&code, 20, 2); + AppendVMovLiteral(&code, 1, 0x0000bbccu); + code.push_back(EncodeFlat0(0x1a, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 1, 20)); + AppendVMovU32(&code, 20, 4); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + code.push_back(EncodeFlat0(0x1d, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 2, 20)); + AppendVMovU32(&code, 20, 12); + AppendVMovLiteral(&code, 4, 0x33333333u); + AppendVMovLiteral(&code, 5, 0x44444444u); + AppendVMovLiteral(&code, 6, 0x55555555u); + code.push_back(EncodeFlat0(0x1f, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 4, 20)); + AppendVMovU32(&code, 20, 24); + AppendVMovLiteral(&code, 7, 0x66666666u); + AppendVMovLiteral(&code, 8, 0x77777777u); + AppendVMovLiteral(&code, 9, 0x88888888u); + AppendVMovLiteral(&code, 10, 0x99999999u); + code.push_back(EncodeFlat0(0x1e, 0, 0)); + code.push_back(EncodeFlat1(0, 0x7d, 7, 20)); + AppendVMovU32(&code, 20, 0); + code.push_back(EncodeFlat0(0x1c, 0, 40)); + code.push_back(EncodeFlat1(0, 0x7d, 2, 20)); + AppendEnd(&code); - TestCase test {"FlatStoreVariants", - code, - std::vector(12, 0), - {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, - 0x66666666u, 0x77777777u, 0x88888888u, 0x99999999u, 0x11111111u, 0}, - {O::VMovB32, O::FlatStoreByte, O::FlatStoreShort, O::FlatStoreDword, - O::FlatStoreDwordx2, O::FlatStoreDwordx3, O::FlatStoreDwordx4, O::SEndpgm}}; - test.flat_memory_base = 0; - return test; + TestCase test{"FlatStoreVariants", + code, + std::vector(12, 0), + {0xbbccaa00u, 0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u, 0x66666666u, 0x77777777u, + 0x88888888u, 0x99999999u, 0x11111111u, 0}, + {O::VMovB32, O::FlatStoreByte, O::FlatStoreShort, + O::FlatStoreDword, O::FlatStoreDwordx2, O::FlatStoreDwordx3, + O::FlatStoreDwordx4, O::SEndpgm}}; + test.flat_memory_base = 0; + return test; } TestCase DsReadWriteVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - AppendVMovLiteral(&code, 2, 0x11223344u); - code.push_back(EncodeDs0(0x0d, 0)); - code.push_back(EncodeDs1(0, 2, 1)); - code.push_back(EncodeDs0(0x36, 0)); - code.push_back(EncodeDs1(3, 0, 1)); - AppendVMovU32(&code, 4, 0xaa); - code.push_back(EncodeDs0(0x1e, 4)); - code.push_back(EncodeDs1(0, 4, 1)); - code.push_back(EncodeDs0(0x3a, 4)); - code.push_back(EncodeDs1(5, 0, 1)); - code.push_back(EncodeDs0(0x39, 4)); - code.push_back(EncodeDs1(6, 0, 1)); - AppendVMovLiteral(&code, 7, 0x000080ffu); - code.push_back(EncodeDs0(0x1f, 8)); - code.push_back(EncodeDs1(0, 7, 1)); - code.push_back(EncodeDs0(0x3c, 8)); - code.push_back(EncodeDs1(8, 0, 1)); - code.push_back(EncodeDs0(0x3b, 8)); - code.push_back(EncodeDs1(9, 0, 1)); - AppendVMovLiteral(&code, 10, 0x10101010u); - AppendVMovLiteral(&code, 11, 0x11111111u); - code.push_back(EncodeDs0(0x4d, 12)); - code.push_back(EncodeDs1(0, 10, 1)); - code.push_back(EncodeDs0(0x76, 12)); - code.push_back(EncodeDs1(14, 0, 1)); - AppendVMovLiteral(&code, 16, 0x20202020u); - AppendVMovLiteral(&code, 17, 0x21212121u); - AppendVMovLiteral(&code, 18, 0x22222222u); - code.push_back(EncodeDs0(0xde, 20)); - code.push_back(EncodeDs1(0, 16, 1)); - code.push_back(EncodeDs0(0xfe, 20)); - code.push_back(EncodeDs1(19, 0, 1)); - AppendVMovLiteral(&code, 22, 0x30303030u); - AppendVMovLiteral(&code, 23, 0x31313131u); - AppendVMovLiteral(&code, 24, 0x32323232u); - AppendVMovLiteral(&code, 25, 0x33333333u); - code.push_back(EncodeDs0(0xdf, 32)); - code.push_back(EncodeDs1(0, 22, 1)); - code.push_back(EncodeDs0(0xff, 32)); - code.push_back(EncodeDs1(26, 0, 1)); + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11223344u); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(3, 0, 1)); + AppendVMovU32(&code, 4, 0xaa); + code.push_back(EncodeDs0(0x1e, 4)); + code.push_back(EncodeDs1(0, 4, 1)); + code.push_back(EncodeDs0(0x3a, 4)); + code.push_back(EncodeDs1(5, 0, 1)); + code.push_back(EncodeDs0(0x39, 4)); + code.push_back(EncodeDs1(6, 0, 1)); + AppendVMovLiteral(&code, 7, 0x000080ffu); + code.push_back(EncodeDs0(0x1f, 8)); + code.push_back(EncodeDs1(0, 7, 1)); + code.push_back(EncodeDs0(0x3c, 8)); + code.push_back(EncodeDs1(8, 0, 1)); + code.push_back(EncodeDs0(0x3b, 8)); + code.push_back(EncodeDs1(9, 0, 1)); + AppendVMovLiteral(&code, 10, 0x10101010u); + AppendVMovLiteral(&code, 11, 0x11111111u); + code.push_back(EncodeDs0(0x4d, 12)); + code.push_back(EncodeDs1(0, 10, 1)); + code.push_back(EncodeDs0(0x76, 12)); + code.push_back(EncodeDs1(14, 0, 1)); + AppendVMovLiteral(&code, 16, 0x20202020u); + AppendVMovLiteral(&code, 17, 0x21212121u); + AppendVMovLiteral(&code, 18, 0x22222222u); + code.push_back(EncodeDs0(0xde, 20)); + code.push_back(EncodeDs1(0, 16, 1)); + code.push_back(EncodeDs0(0xfe, 20)); + code.push_back(EncodeDs1(19, 0, 1)); + AppendVMovLiteral(&code, 22, 0x30303030u); + AppendVMovLiteral(&code, 23, 0x31313131u); + AppendVMovLiteral(&code, 24, 0x32323232u); + AppendVMovLiteral(&code, 25, 0x33333333u); + code.push_back(EncodeDs0(0xdf, 32)); + code.push_back(EncodeDs1(0, 22, 1)); + code.push_back(EncodeDs0(0xff, 32)); + code.push_back(EncodeDs1(26, 0, 1)); - const u32 results[] = {3, 5, 6, 8, 9, 14, 15, 19, 20, 21, 26, 27, 28, 29}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {3, 5, 6, 8, 9, 14, 15, 19, 20, 21, 26, 27, 28, 29}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"DsReadWriteVariants", - code, - std::vector(14, 0), - {0x11223344u, 0xaau, 0xffffffaau, 0x80ffu, 0xffff80ffu, 0x10101010u, 0x11111111u, - 0x20202020u, 0x21212121u, 0x22222222u, 0x30303030u, 0x31313131u, 0x32323232u, - 0x33333333u}, - {O::VMovB32, O::DsWriteB32, O::DsReadB32, O::DsWriteByte, O::DsReadUbyte, - O::DsReadSbyte, O::DsWriteShort, O::DsReadUshort, O::DsReadSshort, O::DsWriteB64, - O::DsReadB64, O::DsWriteB96, O::DsReadB96, O::DsWriteB128, O::DsReadB128, - O::BufferStoreDword, O::SEndpgm}}; + return {"DsReadWriteVariants", + code, + std::vector(14, 0), + {0x11223344u, 0xaau, 0xffffffaau, 0x80ffu, 0xffff80ffu, 0x10101010u, + 0x11111111u, 0x20202020u, 0x21212121u, 0x22222222u, 0x30303030u, + 0x31313131u, 0x32323232u, 0x33333333u}, + {O::VMovB32, O::DsWriteB32, O::DsReadB32, O::DsWriteByte, + O::DsReadUbyte, O::DsReadSbyte, O::DsWriteShort, O::DsReadUshort, + O::DsReadSshort, O::DsWriteB64, O::DsReadB64, O::DsWriteB96, + O::DsReadB96, O::DsWriteB128, O::DsReadB128, O::BufferStoreDword, + O::SEndpgm}}; } TestCase DsReadWrite2Variants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - AppendVMovLiteral(&code, 2, 0x11111111u); - AppendVMovLiteral(&code, 3, 0x22222222u); - code.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u)); - code.push_back(EncodeDs1Ex(0, 3, 2, 1)); - code.push_back(EncodeDs0(0x37, (3u << 8u) | 1u)); - code.push_back(EncodeDs1Ex(4, 0, 0, 1)); - AppendVMovLiteral(&code, 6, 0x33333333u); - AppendVMovLiteral(&code, 7, 0x44444444u); - AppendVMovLiteral(&code, 8, 0x55555555u); - AppendVMovLiteral(&code, 9, 0x66666666u); - code.push_back(EncodeDs0(0x4d, 32)); - code.push_back(EncodeDs1(0, 6, 1)); - code.push_back(EncodeDs0(0x4d, 48)); - code.push_back(EncodeDs1(0, 8, 1)); - code.push_back(EncodeDs0(0x77, (6u << 8u) | 4u)); - code.push_back(EncodeDs1Ex(10, 0, 0, 1)); - AppendVMovLiteral(&code, 14, 0x77777777u); - AppendVMovLiteral(&code, 15, 0x88888888u); - code.push_back(EncodeDs0(0x0f, (2u << 8u) | 1u)); - code.push_back(EncodeDs1Ex(0, 15, 14, 1)); - code.push_back(EncodeDs0(0x38, (2u << 8u) | 1u)); - code.push_back(EncodeDs1Ex(16, 0, 0, 1)); - AppendVMovLiteral(&code, 18, 0x99999999u); - AppendVMovLiteral(&code, 19, 0xaaaaaaaau); - AppendVMovLiteral(&code, 20, 0xbbbbbbbbu); - AppendVMovLiteral(&code, 21, 0xccccccccu); - code.push_back(EncodeDs0(0x4e, (10u << 8u) | 8u)); - code.push_back(EncodeDs1Ex(0, 20, 18, 1)); - code.push_back(EncodeDs0(0x77, (10u << 8u) | 8u)); - code.push_back(EncodeDs1Ex(22, 0, 0, 1)); - AppendVMovLiteral(&code, 26, 0xddddddddu); - AppendVMovLiteral(&code, 27, 0xeeeeeeeeu); - AppendVMovLiteral(&code, 28, 0xf0f0f0f0u); - AppendVMovLiteral(&code, 29, 0x12345678u); - code.push_back(EncodeDs0(0x4f, (2u << 8u) | 1u)); - code.push_back(EncodeDs1Ex(0, 28, 26, 1)); - code.push_back(EncodeDs0(0x78, (2u << 8u) | 1u)); - code.push_back(EncodeDs1Ex(34, 0, 0, 1)); + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x11111111u); + AppendVMovLiteral(&code, 3, 0x22222222u); + code.push_back(EncodeDs0(0x0e, (3u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 3, 2, 1)); + code.push_back(EncodeDs0(0x37, (3u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(4, 0, 0, 1)); + AppendVMovLiteral(&code, 6, 0x33333333u); + AppendVMovLiteral(&code, 7, 0x44444444u); + AppendVMovLiteral(&code, 8, 0x55555555u); + AppendVMovLiteral(&code, 9, 0x66666666u); + code.push_back(EncodeDs0(0x4d, 32)); + code.push_back(EncodeDs1(0, 6, 1)); + code.push_back(EncodeDs0(0x4d, 48)); + code.push_back(EncodeDs1(0, 8, 1)); + code.push_back(EncodeDs0(0x77, (6u << 8u) | 4u)); + code.push_back(EncodeDs1Ex(10, 0, 0, 1)); + AppendVMovLiteral(&code, 14, 0x77777777u); + AppendVMovLiteral(&code, 15, 0x88888888u); + code.push_back(EncodeDs0(0x0f, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 15, 14, 1)); + code.push_back(EncodeDs0(0x38, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(16, 0, 0, 1)); + AppendVMovLiteral(&code, 18, 0x99999999u); + AppendVMovLiteral(&code, 19, 0xaaaaaaaau); + AppendVMovLiteral(&code, 20, 0xbbbbbbbbu); + AppendVMovLiteral(&code, 21, 0xccccccccu); + code.push_back(EncodeDs0(0x4e, (10u << 8u) | 8u)); + code.push_back(EncodeDs1Ex(0, 20, 18, 1)); + code.push_back(EncodeDs0(0x77, (10u << 8u) | 8u)); + code.push_back(EncodeDs1Ex(22, 0, 0, 1)); + AppendVMovLiteral(&code, 26, 0xddddddddu); + AppendVMovLiteral(&code, 27, 0xeeeeeeeeu); + AppendVMovLiteral(&code, 28, 0xf0f0f0f0u); + AppendVMovLiteral(&code, 29, 0x12345678u); + code.push_back(EncodeDs0(0x4f, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(0, 28, 26, 1)); + code.push_back(EncodeDs0(0x78, (2u << 8u) | 1u)); + code.push_back(EncodeDs1Ex(34, 0, 0, 1)); - const u32 results[] = {4, 5, 10, 11, 12, 13, 22, 23, 24, 25, 16, 17, 34, 35, 36, 37}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {4, 5, 10, 11, 12, 13, 22, 23, + 24, 25, 16, 17, 34, 35, 36, 37}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - return {"DsReadWrite2Variants", - code, - std::vector(16, 0), - {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, 0x66666666u, - 0x99999999u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, 0x77777777u, 0x88888888u, - 0xddddddddu, 0xeeeeeeeeu, 0xf0f0f0f0u, 0x12345678u}, - {O::VMovB32, O::DsWrite2B32, O::DsRead2B32, O::DsWriteB64, O::DsRead2B64, - O::DsWrite2B64, O::DsWrite2St64B32, O::DsWrite2St64B64, O::DsRead2St64B64, - O::BufferStoreDword, O::SEndpgm}}; + return {"DsReadWrite2Variants", + code, + std::vector(16, 0), + {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, 0x55555555u, + 0x66666666u, 0x99999999u, 0xaaaaaaaau, 0xbbbbbbbbu, 0xccccccccu, + 0x77777777u, 0x88888888u, 0xddddddddu, 0xeeeeeeeeu, 0xf0f0f0f0u, + 0x12345678u}, + {O::VMovB32, O::DsWrite2B32, O::DsRead2B32, O::DsWriteB64, + O::DsRead2B64, O::DsWrite2B64, O::DsWrite2St64B32, + O::DsWrite2St64B64, O::DsRead2St64B64, O::BufferStoreDword, + O::SEndpgm}}; } TestCase DsAtomicNoReturnVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; - const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu}; - const u32 ops[] = {0x00, 0x01, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b}; - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendVMovLiteral(&code, 2, initial[i]); - code.push_back(EncodeDs0(0x0d, i * 4u)); - code.push_back(EncodeDs1(0, 2, 1)); - AppendVMovLiteral(&code, 3, values[i]); - code.push_back(EncodeDs0(ops[i], i * 4u)); - code.push_back(EncodeDs1(0, 3, 1)); - code.push_back(EncodeDs0(0x36, i * 4u)); - code.push_back(EncodeDs1(10u + i, 0, 1)); - } - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendStoreVgpr(&code, 10u + i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 1, 0); + const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10, + 10, 0xf0f0u, 0xf000u, 0xf00fu}; + const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu}; + const u32 ops[] = {0x00, 0x01, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b}; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovLiteral(&code, 2, initial[i]); + code.push_back(EncodeDs0(0x0d, i * 4u)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, values[i]); + code.push_back(EncodeDs0(ops[i], i * 4u)); + code.push_back(EncodeDs1(0, 3, 1)); + code.push_back(EncodeDs0(0x36, i * 4u)); + code.push_back(EncodeDs1(10u + i, 0, 1)); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, 10u + i, i); + } + AppendEnd(&code); - return {"DsAtomicNoReturnVariants", - code, - std::vector(9, 0), - {15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u}, - {O::VMovB32, O::DsWriteB32, O::DsAddU32, O::DsSubU32, O::DsMinI32, O::DsMaxI32, - O::DsMinU32, O::DsMaxU32, O::DsAndB32, O::DsOrB32, O::DsXorB32, O::DsReadB32, - O::BufferStoreDword, O::SEndpgm}}; + return {"DsAtomicNoReturnVariants", + code, + std::vector(9, 0), + {15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u}, + {O::VMovB32, O::DsWriteB32, O::DsAddU32, O::DsSubU32, O::DsMinI32, + O::DsMaxI32, O::DsMinU32, O::DsMaxU32, O::DsAndB32, O::DsOrB32, + O::DsXorB32, O::DsReadB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase DsAtomicReturnVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10}; - const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu, 99}; - const u32 ops[] = {0x20, 0x21, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2d}; - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendVMovLiteral(&code, 2, initial[i]); - code.push_back(EncodeDs0(0x0d, i * 4u)); - code.push_back(EncodeDs1(0, 2, 1)); - AppendVMovLiteral(&code, 3, values[i]); - code.push_back(EncodeDs0(ops[i], i * 4u)); - code.push_back(EncodeDs1(10u + i, 3, 1)); - code.push_back(EncodeDs0(0x36, i * 4u)); - code.push_back(EncodeDs1(20u + i, 0, 1)); - } - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendStoreVgpr(&code, 10u + i, i); - } - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendStoreVgpr(&code, 20u + i, i + 10u); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 1, 0); + const u32 initial[] = {10, 10, 0xfffffff0u, 0xfffffff0u, 10, + 10, 0xf0f0u, 0xf000u, 0xf00fu, 10}; + const u32 values[] = {5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu, 99}; + const u32 ops[] = {0x20, 0x21, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2d}; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovLiteral(&code, 2, initial[i]); + code.push_back(EncodeDs0(0x0d, i * 4u)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, values[i]); + code.push_back(EncodeDs0(ops[i], i * 4u)); + code.push_back(EncodeDs1(10u + i, 3, 1)); + code.push_back(EncodeDs0(0x36, i * 4u)); + code.push_back(EncodeDs1(20u + i, 0, 1)); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, 10u + i, i); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, 20u + i, i + 10u); + } + AppendEnd(&code); - return {"DsAtomicReturnVariants", - code, - std::vector(20, 0), - {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10, - 15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 99}, - {O::VMovB32, O::DsWriteB32, O::DsAddRtnU32, O::DsSubRtnU32, O::DsMinRtnI32, - O::DsMaxRtnI32, O::DsMinRtnU32, O::DsMaxRtnU32, O::DsAndRtnB32, O::DsOrRtnB32, - O::DsXorRtnB32, O::DsWrxchgRtnB32, O::DsReadB32, O::BufferStoreDword, O::SEndpgm}}; + return { + "DsAtomicReturnVariants", + code, + std::vector(20, 0), + {10, 10, 0xfffffff0u, 0xfffffff0u, 10, 10, 0xf0f0u, 0xf000u, 0xf00fu, 10, + 15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, 99}, + {O::VMovB32, O::DsWriteB32, O::DsAddRtnU32, O::DsSubRtnU32, + O::DsMinRtnI32, O::DsMaxRtnI32, O::DsMinRtnU32, O::DsMaxRtnU32, + O::DsAndRtnB32, O::DsOrRtnB32, O::DsXorRtnB32, O::DsWrxchgRtnB32, + O::DsReadB32, O::BufferStoreDword, O::SEndpgm}}; } TestCase DsMiscVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - AppendVMovLiteral(&code, 2, 0x40800000u); - code.push_back(EncodeDs0(0x0d, 0)); - code.push_back(EncodeDs1(0, 2, 1)); - AppendVMovLiteral(&code, 3, 0x3f800000u); - code.push_back(EncodeDs0(0x0d, 4)); - code.push_back(EncodeDs1(0, 3, 1)); - AppendVMovLiteral(&code, 4, 0x40000000u); - code.push_back(EncodeDs0(0x12, 0)); - code.push_back(EncodeDs1(0, 4, 1)); - AppendVMovLiteral(&code, 5, 0x40400000u); - code.push_back(EncodeDs0(0x13, 4)); - code.push_back(EncodeDs1(0, 5, 1)); - code.push_back(EncodeDs0(0x36, 0)); - code.push_back(EncodeDs1(6, 0, 1)); - code.push_back(EncodeDs0(0x36, 4)); - code.push_back(EncodeDs1(7, 0, 1)); - AppendVMovLiteral(&code, 8, 0x12345678u); - code.push_back(EncodeDs0(0x35, 0x001f)); - code.push_back(EncodeDs1(9, 0, 8)); - code.push_back(EncodeSMovB32(124, InlineU32(0))); - AppendVMovLiteral(&code, 10, 0xabcdef01u); - code.push_back(EncodeDs0(0xb0, 8)); - code.push_back(EncodeDs1(0, 10, 0)); - code.push_back(EncodeDs0(0xb1, 8)); - code.push_back(EncodeDs1(11, 0, 0)); + std::vector code; + AppendVMovU32(&code, 0, 0); + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x40800000u); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, 0x3f800000u); + code.push_back(EncodeDs0(0x0d, 4)); + code.push_back(EncodeDs1(0, 3, 1)); + AppendVMovLiteral(&code, 4, 0x40000000u); + code.push_back(EncodeDs0(0x12, 0)); + code.push_back(EncodeDs1(0, 4, 1)); + AppendVMovLiteral(&code, 5, 0x40400000u); + code.push_back(EncodeDs0(0x13, 4)); + code.push_back(EncodeDs1(0, 5, 1)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(6, 0, 1)); + code.push_back(EncodeDs0(0x36, 4)); + code.push_back(EncodeDs1(7, 0, 1)); + AppendVMovLiteral(&code, 8, 0x12345678u); + code.push_back(EncodeDs0(0x35, 0x001f)); + code.push_back(EncodeDs1(9, 0, 8)); + code.push_back(EncodeSMovB32(124, InlineU32(0))); + AppendVMovLiteral(&code, 10, 0xabcdef01u); + code.push_back(EncodeDs0(0xb0, 8)); + code.push_back(EncodeDs1(0, 10, 0)); + code.push_back(EncodeDs0(0xb1, 8)); + code.push_back(EncodeDs1(11, 0, 0)); - const u32 results[] = {6, 7, 9, 11}; - for (u32 i = 0; i < static_cast(std::size(results)); i++) { - AppendStoreVgpr(&code, results[i], i); - } - AppendEnd(&code); + const u32 results[] = {6, 7, 9, 11}; + for (u32 i = 0; i < static_cast(std::size(results)); i++) { + AppendStoreVgpr(&code, results[i], i); + } + AppendEnd(&code); - TestCase test; - test.name = "DsMiscVariants"; - test.code = code; - test.initial = std::vector(4, 0); - test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u}; - test.opcodes = {O::VMovB32, O::SMovB32, O::DsWriteB32, O::DsMinF32, - O::DsMaxF32, O::DsReadB32, O::DsSwizzleB32, O::DsWriteAddtidB32, - O::DsReadAddtidB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 1; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "DsMiscVariants"; + test.code = code; + test.initial = std::vector(4, 0); + test.expected = {0x40000000u, 0x40400000u, 0x12345678u, 0xabcdef01u}; + test.opcodes = {O::VMovB32, O::SMovB32, O::DsWriteB32, + O::DsMinF32, O::DsMaxF32, O::DsReadB32, + O::DsSwizzleB32, O::DsWriteAddtidB32, O::DsReadAddtidB32, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.has_compute_info = true; + return test; } TestCase DsFloatMinMaxUsesSeparateCompareOperand() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 0); - AppendVMovLiteral(&code, 2, 0x40800000u); - code.push_back(EncodeDs0(0x0d, 0)); - code.push_back(EncodeDs1(0, 2, 1)); - AppendVMovLiteral(&code, 3, 0x40800000u); - code.push_back(EncodeDs0(0x0d, 4)); - code.push_back(EncodeDs1(0, 3, 1)); - AppendVMovLiteral(&code, 4, 0x41100000u); - AppendVMovLiteral(&code, 5, 0x40000000u); - code.push_back(EncodeDs0(0x12, 0)); - code.push_back(EncodeDs1Ex(0, 5, 4, 1)); - AppendVMovLiteral(&code, 6, 0x3f800000u); - AppendVMovLiteral(&code, 7, 0x40400000u); - code.push_back(EncodeDs0(0x13, 4)); - code.push_back(EncodeDs1Ex(0, 7, 6, 1)); - code.push_back(EncodeDs0(0x36, 0)); - code.push_back(EncodeDs1(8, 0, 1)); - code.push_back(EncodeDs0(0x36, 4)); - code.push_back(EncodeDs1(9, 0, 1)); - AppendStoreVgpr(&code, 8, 0); - AppendStoreVgpr(&code, 9, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 1, 0); + AppendVMovLiteral(&code, 2, 0x40800000u); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + AppendVMovLiteral(&code, 3, 0x40800000u); + code.push_back(EncodeDs0(0x0d, 4)); + code.push_back(EncodeDs1(0, 3, 1)); + AppendVMovLiteral(&code, 4, 0x41100000u); + AppendVMovLiteral(&code, 5, 0x40000000u); + code.push_back(EncodeDs0(0x12, 0)); + code.push_back(EncodeDs1Ex(0, 5, 4, 1)); + AppendVMovLiteral(&code, 6, 0x3f800000u); + AppendVMovLiteral(&code, 7, 0x40400000u); + code.push_back(EncodeDs0(0x13, 4)); + code.push_back(EncodeDs1Ex(0, 7, 6, 1)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(8, 0, 1)); + code.push_back(EncodeDs0(0x36, 4)); + code.push_back(EncodeDs1(9, 0, 1)); + AppendStoreVgpr(&code, 8, 0); + AppendStoreVgpr(&code, 9, 1); + AppendEnd(&code); - TestCase test; - test.name = "DsFloatMinMaxUsesSeparateCompareOperand"; - test.code = code; - test.initial = std::vector(2, 0); - test.expected = {0x41100000u, 0x3f800000u}; - test.opcodes = {O::VMovB32, O::DsWriteB32, O::DsMinF32, O::DsMaxF32, - O::DsReadB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 1; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "DsFloatMinMaxUsesSeparateCompareOperand"; + test.code = code; + test.initial = std::vector(2, 0); + test.expected = {0x41100000u, 0x3f800000u}; + test.opcodes = {O::VMovB32, O::DsWriteB32, O::DsMinF32, O::DsMaxF32, + O::DsReadB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.has_compute_info = true; + return test; } TestCase DsSwizzleInvalidSourceLaneZero() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 1, 100); - code.push_back(EncodeDs0(0x35, 0x00e0)); - code.push_back(EncodeDs1(2, 0, 1)); - code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 1, 100); + code.push_back(EncodeDs0(0x35, 0x00e0)); + code.push_back(EncodeDs1(2, 0, 1)); + code.push_back(EncodeVop2(0x1a, 3, InlineU32(2), 0)); + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "DsSwizzleInvalidSourceLaneZero"; - test.code = code; - test.expected = {0, 0, 0, 0}; - test.opcodes = {O::VMovB32, O::DsSwizzleB32, O::VLshlrevB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "DsSwizzleInvalidSourceLaneZero"; + test.code = code; + test.expected = {0, 0, 0, 0}; + test.opcodes = {O::VMovB32, O::DsSwizzleB32, O::VLshlrevB32, + O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase BufferAtomicVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - const u32 values[] = {100, 5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu}; - const u32 ops[] = {0x30, 0x32, 0x33, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b}; - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendVMovU32(&code, 20, i * 4u); - AppendVMovU32(&code, i, values[i]); - AppendBufferStoreOpcode(&code, ops[i], i, 20, true); - } - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendStoreVgpr(&code, i, i + 10u); - } - AppendEnd(&code); + std::vector code; + const u32 values[] = {100, 5, 3, 5, 5, 5, 20, 0x0ff0u, 0x0f00u, 0x00ffu}; + const u32 ops[] = {0x30, 0x32, 0x33, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b}; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovU32(&code, 20, i * 4u); + AppendVMovU32(&code, i, values[i]); + AppendBufferStoreOpcode(&code, ops[i], i, 20, true); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, i, i + 10u); + } + AppendEnd(&code); - return {"BufferAtomicVariants", - code, - {10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {100, 15, 7, 0xfffffff0u, 5, 5, 20, 0x00f0u, 0xff00u, 0xf0f0u, - 10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu}, - {O::VMovB32, O::BufferAtomicSwap, O::BufferAtomicAdd, O::BufferAtomicSub, - O::BufferAtomicSMin, O::BufferAtomicUMin, O::BufferAtomicSMax, O::BufferAtomicUMax, - O::BufferAtomicAnd, O::BufferAtomicOr, O::BufferAtomicXor, O::BufferStoreDword, - O::SEndpgm}}; + return { + "BufferAtomicVariants", + code, + {10, 10, 10, 0xfffffff0u, 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {100, 15, 7, 0xfffffff0u, 5, 5, 20, + 0x00f0u, 0xff00u, 0xf0f0u, 10, 10, 10, 0xfffffff0u, + 10, 0xfffffff0u, 10, 0xf0f0u, 0xf000u, 0xf00fu}, + {O::VMovB32, O::BufferAtomicSwap, O::BufferAtomicAdd, O::BufferAtomicSub, + O::BufferAtomicSMin, O::BufferAtomicUMin, O::BufferAtomicSMax, + O::BufferAtomicUMax, O::BufferAtomicAnd, O::BufferAtomicOr, + O::BufferAtomicXor, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferAtomicGlc0DoesNotReturnOldValue() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 0, 5); - AppendBufferStoreOpcode(&code, 0x32, 0, 20); - AppendStoreVgpr(&code, 0, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 0, 5); + AppendBufferStoreOpcode(&code, 0x32, 0, 20); + AppendStoreVgpr(&code, 0, 1); + AppendEnd(&code); - return {"BufferAtomicGlc0DoesNotReturnOldValue", - code, - {10, 0}, - {15, 5}, - {O::VMovB32, O::BufferAtomicAdd, O::BufferStoreDword, O::SEndpgm}}; + return {"BufferAtomicGlc0DoesNotReturnOldValue", + code, + {10, 0}, + {15, 5}, + {O::VMovB32, O::BufferAtomicAdd, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferAtomicFMinExactRawGlcModes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x40000000u); // 2.0 - code.push_back(0xe0fc0000u); - code.push_back(0x80010000u); // exact failing buffer_atomic_fmin, GLC=0 - AppendStoreVgpr(&code, 0, 1); - AppendVMovLiteral(&code, 0, 0x3f800000u); // 1.0 - code.push_back(0xe0fc4000u); - code.push_back(0x80010000u); // same instruction with GLC=1 - AppendStoreVgpr(&code, 0, 2); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x40000000u); // 2.0 + code.push_back(0xe0fc0000u); + code.push_back(0x80010000u); // exact failing buffer_atomic_fmin, GLC=0 + AppendStoreVgpr(&code, 0, 1); + AppendVMovLiteral(&code, 0, 0x3f800000u); // 1.0 + code.push_back(0xe0fc4000u); + code.push_back(0x80010000u); // same instruction with GLC=1 + AppendStoreVgpr(&code, 0, 2); + AppendEnd(&code); - TestCase test; - test.name = "BufferAtomicFMinExactRawGlcModes"; - test.code = code; - test.initial = {0x40800000u, 0, 0}; // 4.0 - test.expected = {0x3f800000u, 0x40000000u, 0x40000000u}; - test.opcodes = {O::VMovB32, O::BufferAtomicFMin, O::BufferStoreDword, O::SEndpgm}; - const auto descriptor = - MakeStructuredStorageBufferData(0, static_cast(test.initial.size() * sizeof(u32))); - std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4); - test.user_data[50] = 1u << 20u; - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferAtomicFMinExactRawGlcModes"; + test.code = code; + test.initial = {0x40800000u, 0, 0}; // 4.0 + test.expected = {0x3f800000u, 0x40000000u, 0x40000000u}; + test.opcodes = {O::VMovB32, O::BufferAtomicFMin, O::BufferStoreDword, + O::SEndpgm}; + const auto descriptor = MakeStructuredStorageBufferData( + 0, static_cast(test.initial.size() * sizeof(u32))); + std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4); + test.user_data[50] = 1u << 20u; + test.has_user_data = true; + return test; } TestCase BufferAtomicFMinSpecialValues() { - using O = ShaderOpcode; + using O = ShaderOpcode; - const u32 values[] = { - 0x40000000u, // 2.0 - 0x7f800000u, // +infinity - 0xff800000u, // -infinity - 0x3f800000u, // 1.0 - 0x7fc00000u, // quiet NaN - 0x00000000u, // +0.0 - 0x80000000u, // -0.0 - 0x00000000u, // +0.0 - 0x80000001u, // smallest negative denorm - }; - std::vector code; - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendVMovU32(&code, 20, i * 4u); - AppendVMovLiteral(&code, i, values[i]); - AppendBufferStoreOpcode(&code, 0x3f, i, 20, true); - } - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendStoreVgpr(&code, i, i + static_cast(std::size(values))); - } - AppendEnd(&code); + const u32 values[] = { + 0x40000000u, // 2.0 + 0x7f800000u, // +infinity + 0xff800000u, // -infinity + 0x3f800000u, // 1.0 + 0x7fc00000u, // quiet NaN + 0x00000000u, // +0.0 + 0x80000000u, // -0.0 + 0x00000000u, // +0.0 + 0x80000001u, // smallest negative denorm + }; + std::vector code; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovU32(&code, 20, i * 4u); + AppendVMovLiteral(&code, i, values[i]); + AppendBufferStoreOpcode(&code, 0x3f, i, 20, true); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, i, i + static_cast(std::size(values))); + } + AppendEnd(&code); - return {"BufferAtomicFMinSpecialValues", - code, - {0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, - 0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0x40000000u, 0xbf800000u, 0xff800000u, 0x7fc00000u, 0x3f800000u, - 0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u, - 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u, - 0x00000000u, 0x00000001u, 0x00000000u}, - {O::VMovB32, O::BufferAtomicFMin, O::BufferStoreDword, O::SEndpgm}}; + return {"BufferAtomicFMinSpecialValues", + code, + {0x40800000u, 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, + 0x80000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0, 0, 0, 0, 0, 0, + 0, 0, 0}, + {0x40000000u, 0xbf800000u, 0xff800000u, 0x7fc00000u, 0x3f800000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x80000001u, 0x40800000u, + 0xbf800000u, 0x7f800000u, 0x7fc00000u, 0x3f800000u, 0x80000000u, + 0x00000000u, 0x00000001u, 0x00000000u}, + {O::VMovB32, O::BufferAtomicFMin, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferAtomicFMinContendedWorkgroup() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x - AppendVMovU32(&code, 20, 0); - AppendBufferStoreOpcode(&code, 0x3f, 1, 20); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x + AppendVMovU32(&code, 20, 0); + AppendBufferStoreOpcode(&code, 0x3f, 1, 20); + AppendEnd(&code); - TestCase test; - test.name = "BufferAtomicFMinContendedWorkgroup"; - test.code = code; - test.initial = {0x42c80000u}; // 100.0 - test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0) - test.opcodes = {O::VCvtF32U32, O::VMovB32, O::BufferAtomicFMin, O::SEndpgm}; - test.compute_info.threads_num[0] = 64; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "BufferAtomicFMinContendedWorkgroup"; + test.code = code; + test.initial = {0x42c80000u}; // 100.0 + test.expected = {0x00000000u}; // min(100.0, 0.0 .. 63.0) + test.opcodes = {O::VCvtF32U32, O::VMovB32, O::BufferAtomicFMin, O::SEndpgm}; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } TestCase BufferAtomicFMaxExactRawGlcModes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 3, 0x40000000u); // 2.0 - code.push_back(0xe100000cu); - code.push_back(0x80010300u); // exact failing buffer_atomic_fmax v3, offset 12, GLC=0 - AppendStoreVgpr(&code, 3, 4); - AppendVMovLiteral(&code, 0, 0x40800000u); // 4.0 - code.push_back(0xe100400cu); - code.push_back(0x80010000u); // same address with GLC=1 - AppendStoreVgpr(&code, 0, 5); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 3, 0x40000000u); // 2.0 + code.push_back(0xe100000cu); + code.push_back( + 0x80010300u); // exact failing buffer_atomic_fmax v3, offset 12, GLC=0 + AppendStoreVgpr(&code, 3, 4); + AppendVMovLiteral(&code, 0, 0x40800000u); // 4.0 + code.push_back(0xe100400cu); + code.push_back(0x80010000u); // same address with GLC=1 + AppendStoreVgpr(&code, 0, 5); + AppendEnd(&code); - TestCase test; - test.name = "BufferAtomicFMaxExactRawGlcModes"; - test.code = code; - test.initial = {0, 0, 0, 0x3f800000u, 0, 0}; // memory[3] = 1.0 - test.expected = {0, 0, 0, 0x40800000u, 0x40000000u, 0x40000000u}; - test.opcodes = {O::VMovB32, O::BufferAtomicFMax, O::BufferStoreDword, O::SEndpgm}; - const auto descriptor = - MakeStructuredStorageBufferData(0, static_cast(test.initial.size() * sizeof(u32))); - std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4); - test.user_data[50] = 1u << 20u; - test.has_user_data = true; - return test; + TestCase test; + test.name = "BufferAtomicFMaxExactRawGlcModes"; + test.code = code; + test.initial = {0, 0, 0, 0x3f800000u, 0, 0}; // memory[3] = 1.0 + test.expected = {0, 0, 0, 0x40800000u, 0x40000000u, 0x40000000u}; + test.opcodes = {O::VMovB32, O::BufferAtomicFMax, O::BufferStoreDword, + O::SEndpgm}; + const auto descriptor = MakeStructuredStorageBufferData( + 0, static_cast(test.initial.size() * sizeof(u32))); + std::copy_n(descriptor.begin(), 4, test.user_data.begin() + 4); + test.user_data[50] = 1u << 20u; + test.has_user_data = true; + return test; } TestCase BufferAtomicFMaxSpecialValues() { - using O = ShaderOpcode; + using O = ShaderOpcode; - const u32 values[] = { - 0x40000000u, // 2.0, finite update - 0x40000000u, // 2.0, finite no-update - 0x7f800000u, // incoming +infinity - 0xff800000u, // incoming -infinity - 0x3f800000u, // finite against old +infinity - 0x3f800000u, // finite against old -infinity - 0x3f800000u, // finite against old quiet NaN - 0x7fcabcdeu, // incoming quiet NaN - 0x3f800000u, // finite against old signaling NaN - 0x7faabcdeu, // incoming signaling NaN - 0x00000000u, // +0.0 against old -0.0 - 0x80000000u, // -0.0 against old +0.0 - 0x80000000u, // -0.0 against a negative denorm - 0x80000001u, // negative denorm against -0.0 - 0x00000001u, // positive denorm against +0.0 - 0x00000000u, // +0.0 against a positive denorm - }; - std::vector code; - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendVMovU32(&code, 20, i * 4u); - AppendVMovLiteral(&code, i, values[i]); - AppendBufferStoreOpcode(&code, 0x40, i, 20, true); - } - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendStoreVgpr(&code, i, i + static_cast(std::size(values))); - } - AppendEnd(&code); + const u32 values[] = { + 0x40000000u, // 2.0, finite update + 0x40000000u, // 2.0, finite no-update + 0x7f800000u, // incoming +infinity + 0xff800000u, // incoming -infinity + 0x3f800000u, // finite against old +infinity + 0x3f800000u, // finite against old -infinity + 0x3f800000u, // finite against old quiet NaN + 0x7fcabcdeu, // incoming quiet NaN + 0x3f800000u, // finite against old signaling NaN + 0x7faabcdeu, // incoming signaling NaN + 0x00000000u, // +0.0 against old -0.0 + 0x80000000u, // -0.0 against old +0.0 + 0x80000000u, // -0.0 against a negative denorm + 0x80000001u, // negative denorm against -0.0 + 0x00000001u, // positive denorm against +0.0 + 0x00000000u, // +0.0 against a positive denorm + }; + std::vector code; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovU32(&code, 20, i * 4u); + AppendVMovLiteral(&code, i, values[i]); + AppendBufferStoreOpcode(&code, 0x40, i, 20, true); + } + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendStoreVgpr(&code, i, i + static_cast(std::size(values))); + } + AppendEnd(&code); - return {"BufferAtomicFMaxSpecialValues", - code, - {0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u, 0x7f800000u, - 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u, - 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u, 0x00000000u, - 0x00000001u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - {0x40000000u, 0x40800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, - 0x3f800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u, - 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000001u, - 0x00000001u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u, - 0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, - 0x3f800000u, 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u, - 0x00000000u, 0x00000001u}, - {O::VMovB32, O::BufferAtomicFMax, O::BufferStoreDword, O::SEndpgm}}; + return {"BufferAtomicFMaxSpecialValues", + code, + {0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u, + 0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, + 0x7fa54321u, 0x3f800000u, 0x80000000u, 0x00000000u, + 0x80000001u, 0x80000000u, 0x00000000u, 0x00000001u, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0}, + {0x40000000u, 0x40800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, + 0x3f800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, 0x3f800000u, + 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000001u, + 0x00000001u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0x3f800000u, + 0x7f800000u, 0xff800000u, 0x7fc12345u, 0x3f800000u, 0x7fa54321u, + 0x3f800000u, 0x80000000u, 0x00000000u, 0x80000001u, 0x80000000u, + 0x00000000u, 0x00000001u}, + {O::VMovB32, O::BufferAtomicFMax, O::BufferStoreDword, O::SEndpgm}}; } TestCase BufferAtomicFMaxContendedWorkgroup() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x - AppendVMovU32(&code, 20, 0); - AppendBufferStoreOpcode(&code, 0x40, 1, 20); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeVop1(0x06, 1, Vgpr(0))); // v_cvt_f32_u32 v1, thread_id.x + AppendVMovU32(&code, 20, 0); + AppendBufferStoreOpcode(&code, 0x40, 1, 20); + AppendEnd(&code); - TestCase test; - test.name = "BufferAtomicFMaxContendedWorkgroup"; - test.code = code; - test.initial = {0xc2c80000u}; // -100.0 - test.expected = {0x427c0000u}; // max(-100.0, 0.0 .. 63.0) - test.opcodes = {O::VCvtF32U32, O::VMovB32, O::BufferAtomicFMax, O::SEndpgm}; - test.compute_info.threads_num[0] = 64; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - return test; + TestCase test; + test.name = "BufferAtomicFMaxContendedWorkgroup"; + test.code = code; + test.initial = {0xc2c80000u}; // -100.0 + test.expected = {0x427c0000u}; // max(-100.0, 0.0 .. 63.0) + test.opcodes = {O::VCvtF32U32, O::VMovB32, O::BufferAtomicFMax, O::SEndpgm}; + test.compute_info.threads_num[0] = 64; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + return test; } std::vector MakeRgbaImage(u32 width, u32 height, u32 value = 0) { - return std::vector(static_cast(width) * height * 4u, value); + return std::vector(static_cast(width) * height * 4u, value); } -void SetRgbaPixel(std::vector* image, u32 width, u32 x, u32 y, u32 r, u32 g, u32 b, u32 a) { - const auto base = static_cast((y * width + x) * 4u); - (*image)[base + 0] = r; - (*image)[base + 1] = g; - (*image)[base + 2] = b; - (*image)[base + 3] = a; +void SetRgbaPixel(std::vector *image, u32 width, u32 x, u32 y, u32 r, + u32 g, u32 b, u32 a) { + const auto base = static_cast((y * width + x) * 4u); + (*image)[base + 0] = r; + (*image)[base + 1] = g; + (*image)[base + 2] = b; + (*image)[base + 3] = a; } TestCase ImageLoadVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovU32(&code, 21, 1); - AppendVMovU32(&code, 22, 0); - AppendVMovU32(&code, 23, 0); - code.push_back(EncodeMimg0(0x00, 0xf)); - code.push_back(EncodeMimg1(0, 20)); - code.push_back(EncodeMimg0(0x01, 0xf)); - code.push_back(EncodeMimg1(4, 20)); - AppendVMovU32(&code, 24, 0); - code.push_back(EncodeMimg0(0x0e, 0x1)); - code.push_back(EncodeMimg1(8, 24)); - for (u32 i = 0; i < 9u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovU32(&code, 23, 0); + code.push_back(EncodeMimg0(0x00, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeMimg0(0x01, 0xf)); + code.push_back(EncodeMimg1(4, 20)); + AppendVMovU32(&code, 24, 0); + code.push_back(EncodeMimg0(0x0e, 0x1)); + code.push_back(EncodeMimg1(8, 24)); + for (u32 i = 0; i < 9u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - auto image = MakeRgbaImage(4, 4); - SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u); + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); - TestCase test; - test.name = "ImageLoadVariants"; - test.code = code; - test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, - 0x40800000u, 0x3f800000u, 0x40000000u, - 0x40400000u, 0x40800000u, 4}; - test.opcodes = {O::VMovB32, O::ImageLoad, O::ImageLoadMip, - O::ImageGetResinfo, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba = image; - return test; + TestCase test; + test.name = "ImageLoadVariants"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u, 0x3f800000u, 0x40000000u, + 0x40400000u, 0x40800000u, 4}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::ImageLoadMip, + O::ImageGetResinfo, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + return test; } TestCase DsAppendConsumeUsesEncodedLdsSelector() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 124, 0x0000ffffu); - AppendVMovU32(&code, 1, 0); - AppendVMovU32(&code, 2, 10); - code.push_back(EncodeDs0(0x0d, 0)); - code.push_back(EncodeDs1(0, 2, 1)); - code.push_back(EncodeDs0(0x3e, 0)); - code.push_back(EncodeDs1(3, 0, 0)); - code.push_back(EncodeDs0(0x3d, 0)); - code.push_back(EncodeDs1(4, 0, 0)); - AppendSMovLiteral(&code, 124, 0); - code.push_back(EncodeDs0(0x3e, 0)); - code.push_back(EncodeDs1(5, 0, 0)); - code.push_back(EncodeDs0(0x36, 0)); - code.push_back(EncodeDs1(6, 0, 1)); - AppendStoreVgpr(&code, 3, 0); - AppendStoreVgpr(&code, 4, 1); - AppendStoreVgpr(&code, 5, 2); - AppendStoreVgpr(&code, 6, 3); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 124, 0x0000ffffu); + AppendVMovU32(&code, 1, 0); + AppendVMovU32(&code, 2, 10); + code.push_back(EncodeDs0(0x0d, 0)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeDs0(0x3e, 0)); + code.push_back(EncodeDs1(3, 0, 0)); + code.push_back(EncodeDs0(0x3d, 0)); + code.push_back(EncodeDs1(4, 0, 0)); + AppendSMovLiteral(&code, 124, 0); + code.push_back(EncodeDs0(0x3e, 0)); + code.push_back(EncodeDs1(5, 0, 0)); + code.push_back(EncodeDs0(0x36, 0)); + code.push_back(EncodeDs1(6, 0, 1)); + AppendStoreVgpr(&code, 3, 0); + AppendStoreVgpr(&code, 4, 1); + AppendStoreVgpr(&code, 5, 2); + AppendStoreVgpr(&code, 6, 3); + AppendEnd(&code); - return {"DsAppendConsumeLdsSelector", - code, - {}, - {10, 74, 0, 10}, - {O::SMovB32, O::VMovB32, O::DsWriteB32, O::DsReadB32, O::DsAppend, O::DsConsume, - O::BufferStoreDword, O::SEndpgm}}; + return {"DsAppendConsumeLdsSelector", + code, + {}, + {10, 74, 0, 10}, + {O::SMovB32, O::VMovB32, O::DsWriteB32, O::DsReadB32, O::DsAppend, + O::DsConsume, O::BufferStoreDword, O::SEndpgm}}; } TestCase DsAppendUsesEncodedGdsSelector() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendSMovLiteral(&code, 124, 0x00000008u); - code.push_back(EncodeDs0(0x3e, 0, true)); - code.push_back(EncodeDs1(0, 0, 0)); - code.push_back(EncodeDs0(0x3d, 0, true)); - code.push_back(EncodeDs1(1, 0, 0)); - code.push_back(EncodeDs0(0x3e, 4, true)); - code.push_back(EncodeDs1(2, 0, 0)); - code.push_back(EncodeDs0(0x3d, 4, true)); - code.push_back(EncodeDs1(3, 0, 0)); - AppendSMovLiteral(&code, 124, 0x00080008u); - code.push_back(EncodeDs0(0x3e, 4, true)); - code.push_back(EncodeDs1(4, 0, 0)); - code.push_back(EncodeDs0(0x3d, 4, true)); - code.push_back(EncodeDs1(5, 0, 0)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendStoreVgpr(&code, 2, 2); - AppendStoreVgpr(&code, 3, 3); - AppendStoreVgpr(&code, 4, 4); - AppendStoreVgpr(&code, 5, 5); - AppendEnd(&code); + std::vector code; + AppendSMovLiteral(&code, 124, 0x00000008u); + code.push_back(EncodeDs0(0x3e, 0, true)); + code.push_back(EncodeDs1(0, 0, 0)); + code.push_back(EncodeDs0(0x3d, 0, true)); + code.push_back(EncodeDs1(1, 0, 0)); + code.push_back(EncodeDs0(0x3e, 4, true)); + code.push_back(EncodeDs1(2, 0, 0)); + code.push_back(EncodeDs0(0x3d, 4, true)); + code.push_back(EncodeDs1(3, 0, 0)); + AppendSMovLiteral(&code, 124, 0x00080008u); + code.push_back(EncodeDs0(0x3e, 4, true)); + code.push_back(EncodeDs1(4, 0, 0)); + code.push_back(EncodeDs0(0x3d, 4, true)); + code.push_back(EncodeDs1(5, 0, 0)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendStoreVgpr(&code, 2, 2); + AppendStoreVgpr(&code, 3, 3); + AppendStoreVgpr(&code, 4, 4); + AppendStoreVgpr(&code, 5, 5); + AppendEnd(&code); - TestCase test {"DsAppendGdsSelector", - code, - {}, - {10, 74, 20, 84, 40, 104}, - {O::SMovB32, O::DsAppend, O::DsConsume, O::BufferStoreDword, O::SEndpgm}}; - test.gds_initial = {10, 20, 30, 40}; - test.expected_gds = {10, 20, 30, 40}; - return test; + TestCase test{ + "DsAppendGdsSelector", + code, + {}, + {10, 74, 20, 84, 40, 104}, + {O::SMovB32, O::DsAppend, O::DsConsume, O::BufferStoreDword, O::SEndpgm}}; + test.gds_initial = {10, 20, 30, 40}; + test.expected_gds = {10, 20, 30, 40}; + return test; } TestCase DsGdsSubdwordAndAtomicWrites() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVop1(0x01, 1, Vgpr(0))); - code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); - code.push_back(EncodeDs0(0x1e, 0, true)); - code.push_back(EncodeDs1(0, 2, 1)); - code.push_back(EncodeVop2(0x1a, 8, InlineU32(1), 0)); - code.push_back(EncodeVop2(0x25, 8, InlineU32(12), 8)); - code.push_back(EncodeDs0(0x1f, 0, true)); - code.push_back(EncodeDs1(0, 2, 8)); - AppendVMovU32(&code, 3, 4); - AppendVMovU32(&code, 4, 1); - code.push_back(EncodeDs0(0x00, 0, true)); - code.push_back(EncodeDs1(0, 4, 3)); - AppendVMovU32(&code, 5, 8); - AppendVMovLiteral(&code, 6, 0x40a00000u); - AppendVMovLiteral(&code, 7, 0x40a00000u); - code.push_back(EncodeDs0(0x12, 0, true)); - code.push_back(EncodeDs1Ex(0, 7, 6, 5)); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeVop1(0x01, 1, Vgpr(0))); + code.push_back(EncodeVop2(0x25, 2, InlineU32(1), 0)); + code.push_back(EncodeDs0(0x1e, 0, true)); + code.push_back(EncodeDs1(0, 2, 1)); + code.push_back(EncodeVop2(0x1a, 8, InlineU32(1), 0)); + code.push_back(EncodeVop2(0x25, 8, InlineU32(12), 8)); + code.push_back(EncodeDs0(0x1f, 0, true)); + code.push_back(EncodeDs1(0, 2, 8)); + AppendVMovU32(&code, 3, 4); + AppendVMovU32(&code, 4, 1); + code.push_back(EncodeDs0(0x00, 0, true)); + code.push_back(EncodeDs1(0, 4, 3)); + AppendVMovU32(&code, 5, 8); + AppendVMovLiteral(&code, 6, 0x40a00000u); + AppendVMovLiteral(&code, 7, 0x40a00000u); + code.push_back(EncodeDs0(0x12, 0, true)); + code.push_back(EncodeDs1Ex(0, 7, 6, 5)); + AppendEnd(&code); - TestCase test; - test.name = "DsGdsSubdwordAndAtomics"; - test.code = code; - test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::DsWriteByte, - O::DsWriteShort, O::DsAddU32, O::DsMinF32, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; - test.has_compute_info = true; - test.gds_initial = {0, 0, 0x42c80000u, 0, 0}; - test.expected_gds = {0x04030201u, 4, 0x40a00000u, 0x00020001u, 0x00040003u}; - return test; + TestCase test; + test.name = "DsGdsSubdwordAndAtomics"; + test.code = code; + test.opcodes = {O::VMovB32, O::VAddNcU32, O::VLshlrevB32, O::DsWriteByte, + O::DsWriteShort, O::DsAddU32, O::DsMinF32, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.has_compute_info = true; + test.gds_initial = {0, 0, 0x42c80000u, 0, 0}; + test.expected_gds = {0x04030201u, 4, 0x40a00000u, 0x00020001u, 0x00040003u}; + return test; } TestCase ImageLoadR32UintUsesIntegerSampledImage() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovU32(&code, 21, 1); - code.push_back(EncodeMimg0(0x00, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + code.push_back(EncodeMimg0(0x00, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageLoadR32UintUsesIntegerSampledImage"; - test.code = code; - test.expected = {0xdeadbeefu}; - test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba.resize(16); - test.sampled_image_rgba[6] = 0xdeadbeefu; - test.sampled_image_format = vk::Format::eR32Uint; - test.sampled_image_dwords_per_pixel = 1; - test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); - test.has_user_data = true; - test.required_spirv = {"sampled_uint_2d"}; - return test; + TestCase test; + test.name = "ImageLoadR32UintUsesIntegerSampledImage"; + test.code = code; + test.expected = {0xdeadbeefu}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba.resize(16); + test.sampled_image_rgba[6] = 0xdeadbeefu; + test.sampled_image_format = vk::Format::eR32Uint; + test.sampled_image_dwords_per_pixel = 1; + test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.required_spirv = {"sampled_uint_2d"}; + return test; } TestCase ImageLoad1DUsesScalarCoordinate() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageLoad1DUsesScalarCoordinate"; - test.code = code; - test.expected = {0xdeadbeefu}; - test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; - test.image_width = 4; - test.image_height = 1; - test.sampled_image_rgba = {0, 0, 0xdeadbeefu, 0}; - test.sampled_image_format = vk::Format::eR32Uint; - test.sampled_image_dwords_per_pixel = 1; - test.sampled_image_type = vk::ImageType::e1D; - test.sampled_image_view_type = vk::ImageViewType::e1D; - test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); - test.user_data[3] = static_cast(Prospero::ImageType::kColor1D) << 28u; - test.has_user_data = true; - test.required_spirv = {"sampled_uint_1d"}; - return test; + TestCase test; + test.name = "ImageLoad1DUsesScalarCoordinate"; + test.code = code; + test.expected = {0xdeadbeefu}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.image_width = 4; + test.image_height = 1; + test.sampled_image_rgba = {0, 0, 0xdeadbeefu, 0}; + test.sampled_image_format = vk::Format::eR32Uint; + test.sampled_image_dwords_per_pixel = 1; + test.sampled_image_type = vk::ImageType::e1D; + test.sampled_image_view_type = vk::ImageViewType::e1D; + test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); + test.user_data[3] = static_cast(Prospero::ImageType::kColor1D) + << 28u; + test.has_user_data = true; + test.required_spirv = {"sampled_uint_1d"}; + return test; } TestCase ImageLoad1DArrayUsesLayerCoordinate() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovU32(&code, 21, 1); - code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 4)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 4)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageLoad1DArrayUsesLayerCoordinate"; - test.code = code; - test.expected = {0xcafebabeu}; - test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; - test.image_width = 4; - test.image_height = 1; - test.sampled_image_rgba = {0, 0, 0, 0, 0, 0, 0xcafebabeu, 0}; - test.sampled_image_format = vk::Format::eR32Uint; - test.sampled_image_dwords_per_pixel = 1; - test.sampled_image_type = vk::ImageType::e1D; - test.sampled_image_view_type = vk::ImageViewType::e1DArray; - test.sampled_image_layers = 2; - test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); - test.user_data[3] = static_cast(Prospero::ImageType::kColor1DArray) << 28u; - test.has_user_data = true; - test.required_spirv = {"sampled_uint_1d_array"}; - return test; + TestCase test; + test.name = "ImageLoad1DArrayUsesLayerCoordinate"; + test.code = code; + test.expected = {0xcafebabeu}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.image_width = 4; + test.image_height = 1; + test.sampled_image_rgba = {0, 0, 0, 0, 0, 0, 0xcafebabeu, 0}; + test.sampled_image_format = vk::Format::eR32Uint; + test.sampled_image_dwords_per_pixel = 1; + test.sampled_image_type = vk::ImageType::e1D; + test.sampled_image_view_type = vk::ImageViewType::e1DArray; + test.sampled_image_layers = 2; + test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); + test.user_data[3] = static_cast(Prospero::ImageType::kColor1DArray) + << 28u; + test.has_user_data = true; + test.required_spirv = {"sampled_uint_1d_array"}; + return test; } TestCase ImageLoad1DArrayDescriptorUsesSelectedLayer() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + code.push_back(EncodeMimg0(0x00, 0x1, 0, false, 0)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageLoad1DArrayDescriptorUsesSelectedLayer"; - test.code = code; - test.expected = {0xcafebabeu}; - test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; - test.image_width = 4; - test.image_height = 1; - test.sampled_image_rgba = { - 0, 0, 0xdeadbeefu, 0, 0, 0, 0xcafebabeu, 0, - }; - test.sampled_image_format = vk::Format::eR32Uint; - test.sampled_image_dwords_per_pixel = 1; - test.sampled_image_type = vk::ImageType::e1D; - test.sampled_image_view_type = vk::ImageViewType::e1D; - test.sampled_image_layers = 2; - test.sampled_image_view_base_layer = 1; - test.sampled_image_view_layers = 1; - test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); - test.user_data[3] = static_cast(Prospero::ImageType::kColor1DArray) << 28u; - test.user_data[4] = 1u | (1u << 16u); - test.has_user_data = true; - test.required_spirv = {"sampled_uint_1d"}; - return test; + TestCase test; + test.name = "ImageLoad1DArrayDescriptorUsesSelectedLayer"; + test.code = code; + test.expected = {0xcafebabeu}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.image_width = 4; + test.image_height = 1; + test.sampled_image_rgba = { + 0, 0, 0xdeadbeefu, 0, 0, 0, 0xcafebabeu, 0, + }; + test.sampled_image_format = vk::Format::eR32Uint; + test.sampled_image_dwords_per_pixel = 1; + test.sampled_image_type = vk::ImageType::e1D; + test.sampled_image_view_type = vk::ImageViewType::e1D; + test.sampled_image_layers = 2; + test.sampled_image_view_base_layer = 1; + test.sampled_image_view_layers = 1; + test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k32UInt); + test.user_data[3] = static_cast(Prospero::ImageType::kColor1DArray) + << 28u; + test.user_data[4] = 1u | (1u << 16u); + test.has_user_data = true; + test.required_spirv = {"sampled_uint_1d"}; + return test; } TestCase ImageLoadMipUsesVaddr2Lod2D() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 21, 1); - AppendVMovU32(&code, 22, 1); - AppendVMovU32(&code, 23, 0); - code.push_back(EncodeMimg0(0x01, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 1); + AppendVMovU32(&code, 23, 0); + code.push_back(EncodeMimg0(0x01, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - auto base = MakeRgbaImage(4, 4); - SetRgbaPixel(&base, 4, 1, 1, 0x3f800000u, 0, 0, 0); - auto mip1 = MakeRgbaImage(2, 2); - SetRgbaPixel(&mip1, 2, 1, 1, 0x40000000u, 0, 0, 0); + auto base = MakeRgbaImage(4, 4); + SetRgbaPixel(&base, 4, 1, 1, 0x3f800000u, 0, 0, 0); + auto mip1 = MakeRgbaImage(2, 2); + SetRgbaPixel(&mip1, 2, 1, 1, 0x40000000u, 0, 0, 0); - TestCase test; - test.name = "ImageLoadMipUsesVaddr2Lod2D"; - test.code = code; - test.expected = {0x40000000u}; - test.opcodes = {O::VMovB32, O::ImageLoadMip, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba_mips = {base, mip1}; - return test; + TestCase test; + test.name = "ImageLoadMipUsesVaddr2Lod2D"; + test.code = code; + test.expected = {0x40000000u}; + test.opcodes = {O::VMovB32, O::ImageLoadMip, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba_mips = {base, mip1}; + return test; } TestCase ImageLoadMipNsaUsesSelectedAddressVgprs() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - AppendVMovU32(&code, 30, 1); - AppendVMovU32(&code, 31, 1); - code.push_back(EncodeMimg0(0x01, 0x1, 1)); - code.push_back(EncodeMimg1(0, 20)); - code.push_back((30u << 0u) | (31u << 8u)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + AppendVMovU32(&code, 30, 1); + AppendVMovU32(&code, 31, 1); + code.push_back(EncodeMimg0(0x01, 0x1, 1)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back((30u << 0u) | (31u << 8u)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - auto base = MakeRgbaImage(4, 4); - SetRgbaPixel(&base, 4, 1, 0, 0x3f800000u, 0, 0, 0); - auto mip1 = MakeRgbaImage(2, 2); - SetRgbaPixel(&mip1, 2, 1, 1, 0x40a00000u, 0, 0, 0); + auto base = MakeRgbaImage(4, 4); + SetRgbaPixel(&base, 4, 1, 0, 0x3f800000u, 0, 0, 0); + auto mip1 = MakeRgbaImage(2, 2); + SetRgbaPixel(&mip1, 2, 1, 1, 0x40a00000u, 0, 0, 0); - TestCase test; - test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs"; - test.code = code; - test.expected = {0x40a00000u}; - test.opcodes = {O::VMovB32, O::ImageLoadMip, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba_mips = {base, mip1}; - return test; + TestCase test; + test.name = "ImageLoadMipNsaUsesSelectedAddressVgprs"; + test.code = code; + test.expected = {0x40a00000u}; + test.opcodes = {O::VMovB32, O::ImageLoadMip, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba_mips = {base, mip1}; + return test; } TestCase ImageLoadA16UintCoordsOnGpu() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 20, 0x00010002u); // x=2, y=1 packed as u16. - AppendVMovU32(&code, 21, 0); - code.push_back(EncodeMimg0(0x00, 0xf)); - code.push_back(EncodeMimg1(0, 20, 0, 0, true)); - for (u32 i = 0; i < 4u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 20, 0x00010002u); // x=2, y=1 packed as u16. + AppendVMovU32(&code, 21, 0); + code.push_back(EncodeMimg0(0x00, 0xf)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + for (u32 i = 0; i < 4u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - auto image = MakeRgbaImage(4, 4); - SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u); + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); - TestCase test; - test.name = "ImageLoadA16UintCoordsOnGpu"; - test.code = code; - test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba = image; - test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"}; - test.forbidden_spirv = {"UnpackHalf2x16"}; - return test; + TestCase test; + test.name = "ImageLoadA16UintCoordsOnGpu"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::ImageLoad, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + test.required_spirv = {"OpShiftRightLogical", "OpBitwiseAnd"}; + test.forbidden_spirv = {"UnpackHalf2x16"}; + return test; } TestCase ImageGetResinfoDmaskWidthHeight() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 24, 0); - code.push_back(EncodeMimg0(0x0e, 0x3)); - code.push_back(EncodeMimg1(0, 24)); - AppendStoreVgpr(&code, 0, 0); - AppendStoreVgpr(&code, 1, 1); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 24, 0); + code.push_back(EncodeMimg0(0x0e, 0x3)); + code.push_back(EncodeMimg1(0, 24)); + AppendStoreVgpr(&code, 0, 0); + AppendStoreVgpr(&code, 1, 1); + AppendEnd(&code); - TestCase test; - test.name = "ImageGetResinfoDmaskWidthHeight"; - test.code = code; - test.expected = {4, 2}; - test.opcodes = {O::VMovB32, O::ImageGetResinfo, O::BufferStoreDword, O::SEndpgm}; - test.image_width = 4; - test.image_height = 2; - test.sampled_image_rgba = MakeRgbaImage(test.image_width, test.image_height); - return test; + TestCase test; + test.name = "ImageGetResinfoDmaskWidthHeight"; + test.code = code; + test.expected = {4, 2}; + test.opcodes = {O::VMovB32, O::ImageGetResinfo, O::BufferStoreDword, + O::SEndpgm}; + test.image_width = 4; + test.image_height = 2; + test.sampled_image_rgba = MakeRgbaImage(test.image_width, test.image_height); + return test; } TestCase ImageGetResinfoDmaskMipLevels() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 24, 0); - code.push_back(EncodeMimg0(0x0e, 0x8)); - code.push_back(EncodeMimg1(0, 24)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 24, 0); + code.push_back(EncodeMimg0(0x0e, 0x8)); + code.push_back(EncodeMimg1(0, 24)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageGetResinfoDmaskMipLevels"; - test.code = code; - test.expected = {2}; - test.opcodes = {O::VMovB32, O::ImageGetResinfo, O::BufferStoreDword, O::SEndpgm}; - auto base = MakeRgbaImage(4, 4); - auto mip1 = MakeRgbaImage(2, 2); - test.sampled_image_rgba_mips = {base, mip1}; - return test; + TestCase test; + test.name = "ImageGetResinfoDmaskMipLevels"; + test.code = code; + test.expected = {2}; + test.opcodes = {O::VMovB32, O::ImageGetResinfo, O::BufferStoreDword, + O::SEndpgm}; + auto base = MakeRgbaImage(4, 4); + auto mip1 = MakeRgbaImage(2, 2); + test.sampled_image_rgba_mips = {base, mip1}; + return test; } SkippedCase ImageStoreMipWritesExplicitMip2D() { - return {"ImageStoreMipWritesExplicitMip2D", - "requires per-mip storage-image view descriptors; Vulkan " - "OpImageWrite cannot take Lod"}; + return {"ImageStoreMipWritesExplicitMip2D", + "requires per-mip storage-image view descriptors; Vulkan " + "OpImageWrite cannot take Lod"}; } TestCase ImageSampleAndGather() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 20, 0x3f200000u); - AppendVMovLiteral(&code, 21, 0x3ec00000u); - AppendVMovU32(&code, 22, 0); - code.push_back(EncodeMimg0(0x20, 0xf)); - code.push_back(EncodeMimg1(0, 20)); - code.push_back(EncodeMimg0(0x47, 0x1)); - code.push_back(EncodeMimg1(4, 20)); - AppendVMovU32(&code, 24, 0); - AppendVMovLiteral(&code, 25, 0x3f200000u); - AppendVMovLiteral(&code, 26, 0x3ec00000u); - AppendVMovU32(&code, 27, 0); - code.push_back(EncodeMimg0(0x57, 0x1)); - code.push_back(EncodeMimg1(8, 24)); - code.push_back(EncodeMimg0(0x60, 0x1)); - code.push_back(EncodeMimg1(12, 20)); - for (u32 i = 0; i < 13u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 20, 0x3f200000u); + AppendVMovLiteral(&code, 21, 0x3ec00000u); + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMimg0(0x20, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeMimg0(0x47, 0x1)); + code.push_back(EncodeMimg1(4, 20)); + AppendVMovU32(&code, 24, 0); + AppendVMovLiteral(&code, 25, 0x3f200000u); + AppendVMovLiteral(&code, 26, 0x3ec00000u); + AppendVMovU32(&code, 27, 0); + code.push_back(EncodeMimg0(0x57, 0x1)); + code.push_back(EncodeMimg1(8, 24)); + code.push_back(EncodeMimg0(0x60, 0x1)); + code.push_back(EncodeMimg1(12, 20)); + for (u32 i = 0; i < 13u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - auto image = MakeRgbaImage(4, 4); - for (u32 y = 0; y < 4u; y++) { - for (u32 x = 0; x < 4u; x++) { - SetRgbaPixel(&image, 4, x, y, 0x3f800000u, 0, 0, 0); - } - } - SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u); + auto image = MakeRgbaImage(4, 4); + for (u32 y = 0; y < 4u; y++) { + for (u32 x = 0; x < 4u; x++) { + SetRgbaPixel(&image, 4, x, y, 0x3f800000u, 0, 0, 0); + } + } + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); - TestCase test; - test.name = "ImageSampleAndGather"; - test.code = code; - test.expected = {0x3f800000u, - 0x40000000u, - 0x40400000u, - 0x40800000u, - 0x3f800000u, - 0x3f800000u, - 0x3f800000u, - 0x3f800000u, - 0x3f800000u, - 0x3f800000u, - 0x3f800000u, - 0x3f800000u, - 0}; - test.opcodes = {O::VMovB32, O::ImageSample, O::ImageGetLod, O::ImageGather4Lz, - O::ImageGather4LzO, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba = image; - return test; + TestCase test; + test.name = "ImageSampleAndGather"; + test.code = code; + test.expected = {0x3f800000u, + 0x40000000u, + 0x40400000u, + 0x40800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0x3f800000u, + 0}; + test.opcodes = {O::VMovB32, O::ImageSample, O::ImageGetLod, + O::ImageGather4Lz, O::ImageGather4LzO, O::BufferStoreDword, + O::SEndpgm}; + test.sampled_image_rgba = image; + return test; } TestCase ImageSampleA16SamplerCoordsOnGpu() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 20, 0x36003900u); // x=0.625, y=0.375 packed as f16. - AppendVMovU32(&code, 21, 0); - code.push_back(EncodeMimg0(0x20, 0xf)); - code.push_back(EncodeMimg1(0, 20, 0, 0, true)); - for (u32 i = 0; i < 4u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 20, 0x36003900u); // x=0.625, y=0.375 packed as f16. + AppendVMovU32(&code, 21, 0); + code.push_back(EncodeMimg0(0x20, 0xf)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + for (u32 i = 0; i < 4u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - auto image = MakeRgbaImage(4, 4); - SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u); + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); - TestCase test; - test.name = "ImageSampleA16SamplerCoordsOnGpu"; - test.code = code; - test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba = image; - test.required_spirv = {"UnpackHalf2x16"}; - return test; + TestCase test; + test.name = "ImageSampleA16SamplerCoordsOnGpu"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + test.required_spirv = {"UnpackHalf2x16"}; + return test; } TestCase ImageSampleOpcodeAliasUsesNormalCoords() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 2, 0x3f000000u); - AppendVMovLiteral(&code, 3, 0x3f000000u); - code.push_back(0xf0800109u); // observed image_sample_a v6, v2, s0, s24 - code.push_back(0x00c00602u); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 2, 0x3f000000u); + AppendVMovLiteral(&code, 3, 0x3f000000u); + code.push_back(0xf0800109u); // observed image_sample_a v6, v2, s0, s24 + code.push_back(0x00c00602u); + AppendStoreVgpr(&code, 6, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageSampleOpcodeAliasUsesNormalCoords"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageSample, O::SEndpgm}; - test.required_spirv = {"OpImageSampleExplicitLod"}; - test.forbidden_spirv = {"UnpackHalf2x16"}; - test.compile_only = true; - return test; + TestCase test; + test.name = "ImageSampleOpcodeAliasUsesNormalCoords"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpImageSampleExplicitLod"}; + test.forbidden_spirv = {"UnpackHalf2x16"}; + test.compile_only = true; + return test; } TestCase ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, - 1); // Non-constant +1 X offset is not a SPIR-V ConstOffset. - AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16. - AppendVMovU32(&code, 22, 0); - code.push_back(EncodeMimg0(0x30, 0xf)); - code.push_back(EncodeMimg1(0, 20, 0, 0, true)); - for (u32 i = 0; i < 4u; i++) { - AppendStoreVgpr(&code, i, i); - } - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, + 1); // Non-constant +1 X offset is not a SPIR-V ConstOffset. + AppendVMovLiteral(&code, 21, 0x36003900u); // x=0.625, y=0.375 packed as f16. + AppendVMovU32(&code, 22, 0); + code.push_back(EncodeMimg0(0x30, 0xf)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + for (u32 i = 0; i < 4u; i++) { + AppendStoreVgpr(&code, i, i); + } + AppendEnd(&code); - auto image = MakeRgbaImage(4, 4); - SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u); + auto image = MakeRgbaImage(4, 4); + SetRgbaPixel(&image, 4, 2, 1, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); - TestCase test; - test.name = "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu"; - test.code = code; - test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; - test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; - test.sampled_image_rgba = image; - test.required_spirv = {"UnpackHalf2x16"}; - test.forbidden_spirv = {"OpBitFieldSExtract"}; - return test; + TestCase test; + test.name = "ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu"; + test.code = code; + test.expected = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; + test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; + test.sampled_image_rgba = image; + test.required_spirv = {"UnpackHalf2x16"}; + test.forbidden_spirv = {"OpBitFieldSExtract"}; + return test; } TestCase ImageSampleA16CompareBiasRdna2AddressOrder() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 20, 0x00003800u); // bias=0.5 as low f16. - AppendVMovLiteral(&code, 21, 0x3f000000u); // PCF reference stays 32-bit. - AppendVMovLiteral(&code, 22, 0x36003900u); // x=0.625, y=0.375 packed as f16. - code.push_back(EncodeMimg0(0x2d, 0x1)); - code.push_back(EncodeMimg1(0, 20, 0, 0, true)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 20, 0x00003800u); // bias=0.5 as low f16. + AppendVMovLiteral(&code, 21, 0x3f000000u); // PCF reference stays 32-bit. + AppendVMovLiteral(&code, 22, 0x36003900u); // x=0.625, y=0.375 packed as f16. + code.push_back(EncodeMimg0(0x2d, 0x1)); + code.push_back(EncodeMimg1(0, 20, 0, 0, true)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageSampleA16CompareBiasRdna2AddressOrder"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageSample, O::SEndpgm}; - test.required_spirv = {"OpImageSampleDrefExplicitLod", "UnpackHalf2x16"}; - test.compile_only = true; - return test; + TestCase test; + test.name = "ImageSampleA16CompareBiasRdna2AddressOrder"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageSample, O::BufferStoreDword, O::SEndpgm}; + test.required_spirv = {"OpImageSampleDrefExplicitLod", "UnpackHalf2x16"}; + test.compile_only = true; + return test; } TestCase ImageGatherCompareOpcodes() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovLiteral(&code, 21, 0x3f000000u); - AppendVMovLiteral(&code, 22, 0x3f200000u); - AppendVMovLiteral(&code, 23, 0x3ec00000u); - AppendVMovLiteral(&code, 24, 0x3f000000u); - AppendVMovLiteral(&code, 25, 0x3f200000u); - AppendVMovLiteral(&code, 26, 0x3ec00000u); - AppendVMovLiteral(&code, 28, 0x3f000000u); - AppendVMovLiteral(&code, 29, 0x3f200000u); - AppendVMovLiteral(&code, 30, 0x3ec00000u); - AppendVMovU32(&code, 32, 0); - AppendVMovLiteral(&code, 33, 0x3f000000u); - AppendVMovLiteral(&code, 34, 0x3f200000u); - AppendVMovLiteral(&code, 35, 0x3ec00000u); - code.push_back(EncodeMimg0(0x48, 0x1)); - code.push_back(EncodeMimg1(4, 24)); - code.push_back(EncodeMimg0(0x4f, 0x1)); - code.push_back(EncodeMimg1(8, 28)); - code.push_back(EncodeMimg0(0x58, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - code.push_back(EncodeMimg0(0x5f, 0x1)); - code.push_back(EncodeMimg1(12, 32)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovLiteral(&code, 21, 0x3f000000u); + AppendVMovLiteral(&code, 22, 0x3f200000u); + AppendVMovLiteral(&code, 23, 0x3ec00000u); + AppendVMovLiteral(&code, 24, 0x3f000000u); + AppendVMovLiteral(&code, 25, 0x3f200000u); + AppendVMovLiteral(&code, 26, 0x3ec00000u); + AppendVMovLiteral(&code, 28, 0x3f000000u); + AppendVMovLiteral(&code, 29, 0x3f200000u); + AppendVMovLiteral(&code, 30, 0x3ec00000u); + AppendVMovU32(&code, 32, 0); + AppendVMovLiteral(&code, 33, 0x3f000000u); + AppendVMovLiteral(&code, 34, 0x3f200000u); + AppendVMovLiteral(&code, 35, 0x3ec00000u); + code.push_back(EncodeMimg0(0x48, 0x1)); + code.push_back(EncodeMimg1(4, 24)); + code.push_back(EncodeMimg0(0x4f, 0x1)); + code.push_back(EncodeMimg1(8, 28)); + code.push_back(EncodeMimg0(0x58, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + code.push_back(EncodeMimg0(0x5f, 0x1)); + code.push_back(EncodeMimg1(12, 32)); + AppendStoreVgpr(&code, 4, 0); + AppendStoreVgpr(&code, 8, 1); + AppendStoreVgpr(&code, 0, 2); + AppendStoreVgpr(&code, 12, 3); + AppendEnd(&code); - TestCase test; - test.name = "ImageGatherCompareOpcodes"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageGather4C, O::ImageGather4CLz, - O::ImageGather4CO, O::ImageGather4CLzO, O::SEndpgm}; - test.required_spirv = {"OpImageDrefGather", "OpBitFieldSExtract"}; - test.compile_only = true; - return test; + TestCase test; + test.name = "ImageGatherCompareOpcodes"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageGather4C, O::ImageGather4CLz, + O::ImageGather4CO, O::ImageGather4CLzO, O::BufferStoreDword, + O::SEndpgm}; + test.required_spirv = {"OpImageDrefGather", "OpBitFieldSExtract"}; + test.compile_only = true; + return test; } TestCase ImageStoreVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 21, 2); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x40000000u); - AppendVMovLiteral(&code, 2, 0x40400000u); - AppendVMovLiteral(&code, 3, 0x40800000u); - code.push_back(EncodeMimg0(0x08, 0xf)); - code.push_back(EncodeMimg1(0, 20)); - AppendVMovU32(&code, 24, 3); - AppendVMovU32(&code, 25, 2); - AppendVMovU32(&code, 26, 0); - AppendVMovU32(&code, 27, 0); - AppendVMovLiteral(&code, 4, 0x40a00000u); - AppendVMovLiteral(&code, 5, 0x40c00000u); - AppendVMovLiteral(&code, 6, 0x40e00000u); - AppendVMovLiteral(&code, 7, 0x41000000u); - code.push_back(EncodeMimg0(0x09, 0xf)); - code.push_back(EncodeMimg1(4, 24)); - AppendVMovLiteral(&code, 8, 0x12345678u); - AppendStoreVgpr(&code, 8, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x40000000u); + AppendVMovLiteral(&code, 2, 0x40400000u); + AppendVMovLiteral(&code, 3, 0x40800000u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendVMovU32(&code, 24, 3); + AppendVMovU32(&code, 25, 2); + AppendVMovU32(&code, 26, 0); + AppendVMovU32(&code, 27, 0); + AppendVMovLiteral(&code, 4, 0x40a00000u); + AppendVMovLiteral(&code, 5, 0x40c00000u); + AppendVMovLiteral(&code, 6, 0x40e00000u); + AppendVMovLiteral(&code, 7, 0x41000000u); + code.push_back(EncodeMimg0(0x09, 0xf)); + code.push_back(EncodeMimg1(4, 24)); + AppendVMovLiteral(&code, 8, 0x12345678u); + AppendStoreVgpr(&code, 8, 0); + AppendEnd(&code); - auto expected_image = MakeRgbaImage(4, 4); - SetRgbaPixel(&expected_image, 4, 1, 2, 0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u); - SetRgbaPixel(&expected_image, 4, 3, 2, 0x40a00000u, 0x40c00000u, 0x40e00000u, 0x41000000u); + auto expected_image = MakeRgbaImage(4, 4); + SetRgbaPixel(&expected_image, 4, 1, 2, 0x3f800000u, 0x40000000u, 0x40400000u, + 0x40800000u); + SetRgbaPixel(&expected_image, 4, 3, 2, 0x40a00000u, 0x40c00000u, 0x40e00000u, + 0x41000000u); - TestCase test; - test.name = "ImageStoreVariants"; - test.code = code; - test.expected = {0x12345678u}; - test.opcodes = {O::VMovB32, O::ImageStore, O::ImageStoreMip, O::BufferStoreDword, O::SEndpgm}; - test.storage_image_rgba = MakeRgbaImage(4, 4); - test.storage_image_r32ui = std::vector(16, 0); - test.expected_storage_image_rgba = expected_image; - return test; + TestCase test; + test.name = "ImageStoreVariants"; + test.code = code; + test.expected = {0x12345678u}; + test.opcodes = {O::VMovB32, O::ImageStore, O::ImageStoreMip, + O::BufferStoreDword, O::SEndpgm}; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_rgba = expected_image; + return test; } TestCase ImageStoreRgbOneUsesInverseSwizzle() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 21, 2); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x3f008081u); - AppendVMovLiteral(&code, 2, 0x3e808081u); - AppendVMovLiteral(&code, 3, 0x3f40c0c1u); - code.push_back(EncodeMimg0(0x08, 0xf)); - code.push_back(EncodeMimg1(0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x3f008081u); + AppendVMovLiteral(&code, 2, 0x3e808081u); + AppendVMovLiteral(&code, 3, 0x3f40c0c1u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "ImageStoreRgbOneUsesInverseSwizzle"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; - test.has_user_data = true; - test.user_data[0] = 0x1000u; - test.user_data[1] = static_cast(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u; - test.image_descriptor_swizzle = DstSel(4, 5, 6, 1); - test.storage_image_format = vk::Format::eR8G8B8A8Unorm; - test.storage_image_dwords_per_pixel = 1; - test.storage_image_rgba = std::vector(16, 0); - test.expected_storage_image_rgba = std::vector(16, 0); - test.expected_storage_image_rgba[2 * 4 + 1] = 0x004080ffu; - return test; + TestCase test; + test.name = "ImageStoreRgbOneUsesInverseSwizzle"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.has_user_data = true; + test.user_data[0] = 0x1000u; + test.user_data[1] = + static_cast(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u; + test.image_descriptor_swizzle = DstSel(4, 5, 6, 1); + test.storage_image_format = vk::Format::eR8G8B8A8Unorm; + test.storage_image_dwords_per_pixel = 1; + test.storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba[2 * 4 + 1] = 0x004080ffu; + return test; } TestCase ImageStoreDuplicateSelectorUsesInverseSwizzle() { - auto test = ImageStoreRgbOneUsesInverseSwizzle(); - test.name = "ImageStoreDuplicateSelectorUsesInverseSwizzle"; - test.image_descriptor_swizzle = DstSel(4, 4, 6, 7); - test.expected_storage_image_rgba[2 * 4 + 1] = 0xc04000ffu; - return test; + auto test = ImageStoreRgbOneUsesInverseSwizzle(); + test.name = "ImageStoreDuplicateSelectorUsesInverseSwizzle"; + test.image_descriptor_swizzle = DstSel(4, 4, 6, 7); + test.expected_storage_image_rgba[2 * 4 + 1] = 0xc04000ffu; + return test; } TestCase ImageStoreBgraUsesInverseSwizzle() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 21, 2); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x3f008081u); // 128/255 - AppendVMovLiteral(&code, 2, 0x3e808081u); // 64/255 - AppendVMovLiteral(&code, 3, 0x3f3fbfc0u); // 191/255 - code.push_back(EncodeMimg0(0x08, 0xf)); - code.push_back(EncodeMimg1(0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x3f008081u); // 128/255 + AppendVMovLiteral(&code, 2, 0x3e808081u); // 64/255 + AppendVMovLiteral(&code, 3, 0x3f3fbfc0u); // 191/255 + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "ImageStoreBgraUsesInverseSwizzle"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; - test.has_user_data = true; - test.user_data[0] = 0x1000u; - test.user_data[1] = static_cast(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u; - test.image_descriptor_swizzle = DstSel(6, 5, 4, 7); - test.storage_image_format = vk::Format::eR8G8B8A8Unorm; - test.storage_image_dwords_per_pixel = 1; - test.storage_image_rgba = std::vector(16, 0); - test.expected_storage_image_rgba = std::vector(16, 0); - test.expected_storage_image_rgba[2 * 4 + 1] = 0xbfff8040u; - return test; + TestCase test; + test.name = "ImageStoreBgraUsesInverseSwizzle"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.has_user_data = true; + test.user_data[0] = 0x1000u; + test.user_data[1] = + static_cast(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u; + test.image_descriptor_swizzle = DstSel(6, 5, 4, 7); + test.storage_image_format = vk::Format::eR8G8B8A8Unorm; + test.storage_image_dwords_per_pixel = 1; + test.storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba[2 * 4 + 1] = 0xbfff8040u; + return test; } TestCase ImageStoreYzwxUsesInverseSwizzle() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 1); - AppendVMovU32(&code, 21, 2); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x3f800000u); - AppendVMovLiteral(&code, 1, 0x3f000000u); - AppendVMovLiteral(&code, 2, 0x3e800000u); - AppendVMovLiteral(&code, 3, 0x3f400000u); - code.push_back(EncodeMimg0(0x08, 0xf)); - code.push_back(EncodeMimg1(0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 1); + AppendVMovU32(&code, 21, 2); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + AppendVMovLiteral(&code, 1, 0x3f000000u); + AppendVMovLiteral(&code, 2, 0x3e800000u); + AppendVMovLiteral(&code, 3, 0x3f400000u); + code.push_back(EncodeMimg0(0x08, 0xf)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); - TestCase test; - test.name = "ImageStoreYzwxUsesInverseSwizzle"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; - test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float); - test.has_user_data = true; - test.image_descriptor_swizzle = DstSel(5, 6, 7, 4); - test.storage_image_rgba = MakeRgbaImage(4, 4); - test.expected_storage_image_rgba = MakeRgbaImage(4, 4); - SetRgbaPixel(&test.expected_storage_image_rgba, 4, 1, 2, 0x3f400000u, 0x3f800000u, 0x3f000000u, - 0x3e800000u); - return test; + TestCase test; + test.name = "ImageStoreYzwxUsesInverseSwizzle"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.user_data = + MakeStorageTextureData(Prospero::BufferFormat::k32_32_32_32Float); + test.has_user_data = true; + test.image_descriptor_swizzle = DstSel(5, 6, 7, 4); + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.expected_storage_image_rgba = MakeRgbaImage(4, 4); + SetRgbaPixel(&test.expected_storage_image_rgba, 4, 1, 2, 0x3f400000u, + 0x3f800000u, 0x3f000000u, 0x3e800000u); + return test; } TestCase ImageStoreR32FloatUsesFormatlessStorageImage() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovU32(&code, 21, 1); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x3f800000u); - code.push_back(EncodeMimg0(0x08, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x3f800000u); + code.push_back(EncodeMimg0(0x08, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); - std::vector expected_image(16, 0); - expected_image[1 * 4 + 2] = 0x3f800000u; + std::vector expected_image(16, 0); + expected_image[1 * 4 + 2] = 0x3f800000u; - TestCase test; - test.name = "ImageStoreR32FloatUsesFormatlessStorageImage"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; - test.storage_image_format = vk::Format::eR32Sfloat; - test.storage_image_dwords_per_pixel = 1; - test.storage_image_rgba = std::vector(16, 0); - test.expected_storage_image_rgba = expected_image; - test.required_spirv = {"OpCapability StorageImageReadWithoutFormat", - "OpCapability StorageImageWriteWithoutFormat"}; - test.forbidden_spirv = {"Rgba32f"}; - return test; + TestCase test; + test.name = "ImageStoreR32FloatUsesFormatlessStorageImage"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.storage_image_format = vk::Format::eR32Sfloat; + test.storage_image_dwords_per_pixel = 1; + test.storage_image_rgba = std::vector(16, 0); + test.expected_storage_image_rgba = expected_image; + test.required_spirv = {"OpCapability StorageImageReadWithoutFormat", + "OpCapability StorageImageWriteWithoutFormat"}; + test.forbidden_spirv = {"Rgba32f"}; + return test; } TestCase ImageStoreR32SintUsesRawUintView() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovU32(&code, 21, 1); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x80000001u); - code.push_back(EncodeMimg0(0x08, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x80000001u); + code.push_back(EncodeMimg0(0x08, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); - std::vector expected_image(16, 0); - expected_image[1 * 4 + 2] = 0x80000001u; + std::vector expected_image(16, 0); + expected_image[1 * 4 + 2] = 0x80000001u; - TestCase test; - test.name = "ImageStoreR32SintUsesRawUintView"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; - test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32SInt); - test.has_user_data = true; - test.storage_image_r32ui = std::vector(16, 0); - test.expected_storage_image_r32ui = expected_image; - test.required_spirv = {"storage_uint_2d"}; - return test; + TestCase test; + test.name = "ImageStoreR32SintUsesRawUintView"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32SInt); + test.has_user_data = true; + test.storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui = expected_image; + test.required_spirv = {"storage_uint_2d"}; + return test; } TestCase ImageStoreR32UintUsesUintStorageImage() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 2); - AppendVMovU32(&code, 21, 1); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, 0x12345678u); - code.push_back(EncodeMimg0(0x08, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 2); + AppendVMovU32(&code, 21, 1); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, 0x12345678u); + code.push_back(EncodeMimg0(0x08, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendEnd(&code); - std::vector expected_image(16, 0); - expected_image[1 * 4 + 2] = 0x12345678u; + std::vector expected_image(16, 0); + expected_image[1 * 4 + 2] = 0x12345678u; - TestCase test; - test.name = "ImageStoreR32UintUsesUintStorageImage"; - test.code = code; - test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; - test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt); - test.has_user_data = true; - test.storage_image_rgba = MakeRgbaImage(4, 4); - test.storage_image_r32ui = std::vector(16, 0); - test.expected_storage_image_r32ui = expected_image; - test.required_spirv = {"R32ui", "storage_uint_2d"}; - return test; + TestCase test; + test.name = "ImageStoreR32UintUsesUintStorageImage"; + test.code = code; + test.opcodes = {O::VMovB32, O::ImageStore, O::SEndpgm}; + test.user_data = MakeStorageTextureData(Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui = expected_image; + test.required_spirv = {"R32ui", "storage_uint_2d"}; + return test; } TestCase ComputeTgSizeSgprUsesWaveMetadata() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendStoreSgpr(&code, 2, 0); - AppendEnd(&code); + std::vector code; + AppendStoreSgpr(&code, 2, 0); + AppendEnd(&code); - TestCase test; - test.name = "ComputeTgSizeSgprUsesWaveMetadata"; - test.code = code; - test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 4; - test.compute_info.threads_num[2] = 16; - test.compute_info.group_id[0] = true; - test.compute_info.group_id[1] = true; - test.compute_info.wave_size = 32; - test.compute_info.thread_ids_num = 3; - test.compute_info.workgroup_register = 0; - test.compute_info.tg_size_en = true; - test.has_compute_info = true; - test.compile_only = true; - test.required_spirv = {"OpUDiv", "OpShiftLeftLogical", "2147483648"}; - return test; + TestCase test; + test.name = "ComputeTgSizeSgprUsesWaveMetadata"; + test.code = code; + test.opcodes = {O::VMovB32, O::BufferStoreDword, O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 4; + test.compute_info.threads_num[2] = 16; + test.compute_info.group_id[0] = true; + test.compute_info.group_id[1] = true; + test.compute_info.wave_size = 32; + test.compute_info.thread_ids_num = 3; + test.compute_info.workgroup_register = 0; + test.compute_info.tg_size_en = true; + test.has_compute_info = true; + test.compile_only = true; + test.required_spirv = {"OpUDiv", "OpShiftLeftLogical", "2147483648"}; + return test; } TestCase ImageAtomicVariants() { - using O = ShaderOpcode; + using O = ShaderOpcode; - const u32 initial[] = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; - const u32 values[] = {5, 5, 0x0ff0u, 0x0f00u, 0x00ffu}; - const u32 ops[] = {0x11, 0x15, 0x18, 0x19, 0x1a}; + const u32 initial[] = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; + const u32 values[] = {5, 5, 0x0ff0u, 0x0f00u, 0x00ffu}; + const u32 ops[] = {0x11, 0x15, 0x18, 0x19, 0x1a}; - std::vector code; - for (u32 i = 0; i < static_cast(std::size(values)); i++) { - AppendVMovU32(&code, 20, i & 3u); - AppendVMovU32(&code, 21, i >> 2u); - AppendVMovU32(&code, 22, 0); - AppendVMovLiteral(&code, 0, values[i]); - code.push_back(EncodeMimg0(ops[i], 0x1, 0, true)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, i); - } - AppendEnd(&code); + std::vector code; + for (u32 i = 0; i < static_cast(std::size(values)); i++) { + AppendVMovU32(&code, 20, i & 3u); + AppendVMovU32(&code, 21, i >> 2u); + AppendVMovU32(&code, 22, 0); + AppendVMovLiteral(&code, 0, values[i]); + code.push_back(EncodeMimg0(ops[i], 0x1, 0, true)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, i); + } + AppendEnd(&code); - TestCase test; - test.name = "ImageAtomicVariants"; - test.code = code; - test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; - test.opcodes = {O::VMovB32, O::ImageAtomicAdd, O::ImageAtomicUMin, O::ImageAtomicAnd, - O::ImageAtomicOr, O::ImageAtomicXor, O::BufferStoreDword, O::SEndpgm}; - test.storage_image_rgba = MakeRgbaImage(4, 4); - test.storage_image_r32ui = std::vector(16, 0); - for (u32 i = 0; i < static_cast(std::size(initial)); i++) { - test.storage_image_r32ui[i] = initial[i]; - } - test.expected_storage_image_r32ui = std::vector(16, 0); - test.expected_storage_image_r32ui[0] = 15; - test.expected_storage_image_r32ui[1] = 5; - test.expected_storage_image_r32ui[2] = 0x00f0u; - test.expected_storage_image_r32ui[3] = 0xff00u; - test.expected_storage_image_r32ui[4] = 0xf0f0u; - return test; + TestCase test; + test.name = "ImageAtomicVariants"; + test.code = code; + test.expected = {10, 10, 0xf0f0u, 0xf000u, 0xf00fu}; + test.opcodes = {O::VMovB32, O::ImageAtomicAdd, O::ImageAtomicUMin, + O::ImageAtomicAnd, O::ImageAtomicOr, O::ImageAtomicXor, + O::BufferStoreDword, O::SEndpgm}; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + for (u32 i = 0; i < static_cast(std::size(initial)); i++) { + test.storage_image_r32ui[i] = initial[i]; + } + test.expected_storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui[0] = 15; + test.expected_storage_image_r32ui[1] = 5; + test.expected_storage_image_r32ui[2] = 0x00f0u; + test.expected_storage_image_r32ui[3] = 0xff00u; + test.expected_storage_image_r32ui[4] = 0xf0f0u; + return test; } TestCase ImageAtomicGlc0DoesNotReturnOldValue() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovU32(&code, 20, 0); - AppendVMovU32(&code, 21, 0); - AppendVMovU32(&code, 22, 0); - AppendVMovU32(&code, 0, 5); - code.push_back(EncodeMimg0(0x11, 0x1)); - code.push_back(EncodeMimg1(0, 20)); - AppendStoreVgpr(&code, 0, 0); - AppendEnd(&code); + std::vector code; + AppendVMovU32(&code, 20, 0); + AppendVMovU32(&code, 21, 0); + AppendVMovU32(&code, 22, 0); + AppendVMovU32(&code, 0, 5); + code.push_back(EncodeMimg0(0x11, 0x1)); + code.push_back(EncodeMimg1(0, 20)); + AppendStoreVgpr(&code, 0, 0); + AppendEnd(&code); - TestCase test; - test.name = "ImageAtomicGlc0DoesNotReturnOldValue"; - test.code = code; - test.expected = {5}; - test.opcodes = {O::VMovB32, O::ImageAtomicAdd, O::BufferStoreDword, O::SEndpgm}; - test.storage_image_rgba = MakeRgbaImage(4, 4); - test.storage_image_r32ui = std::vector(16, 0); - test.storage_image_r32ui[0] = 10; - test.expected_storage_image_r32ui = std::vector(16, 0); - test.expected_storage_image_r32ui[0] = 15; - return test; + TestCase test; + test.name = "ImageAtomicGlc0DoesNotReturnOldValue"; + test.code = code; + test.expected = {5}; + test.opcodes = {O::VMovB32, O::ImageAtomicAdd, O::BufferStoreDword, + O::SEndpgm}; + test.storage_image_rgba = MakeRgbaImage(4, 4); + test.storage_image_r32ui = std::vector(16, 0); + test.storage_image_r32ui[0] = 10; + test.expected_storage_image_r32ui = std::vector(16, 0); + test.expected_storage_image_r32ui[0] = 15; + return test; } GraphicsCase GraphicsInterpolationExport() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVintrp(0x00, 0, 0, 0, 0)); - code.push_back(EncodeVintrp(0x01, 0, 0, 0, 0)); - code.push_back(EncodeVintrp(0x02, 1, 0, 1, 2)); - AppendVMovLiteral(&code, 2, 0x3f400000u); - AppendVMovLiteral(&code, 3, 0x3f800000u); - code.push_back(EncodeExp0(0x00, 0xf)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeVintrp(0x00, 0, 0, 0, 0)); + code.push_back(EncodeVintrp(0x01, 0, 0, 0, 0)); + code.push_back(EncodeVintrp(0x02, 1, 0, 1, 2)); + AppendVMovLiteral(&code, 2, 0x3f400000u); + AppendVMovLiteral(&code, 3, 0x3f800000u); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + AppendEnd(&code); - return {"GraphicsInterpolationExport", - code, - {0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u}, - {O::VInterpP1F32, O::VInterpP2F32, O::VInterpMovF32, O::VMovB32, O::Exp, O::SEndpgm}}; + return {"GraphicsInterpolationExport", + code, + {0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u}, + {O::VInterpP1F32, O::VInterpP2F32, O::VInterpMovF32, O::VMovB32, + O::Exp, O::SEndpgm}}; } GraphicsCase GraphicsFlatInterpolatorExport() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeVintrp(0x02, 0, 0, 0, 2)); - AppendVMovLiteral(&code, 1, 0x00000000u); - AppendVMovLiteral(&code, 2, 0x00000000u); - AppendVMovLiteral(&code, 3, 0x3f800000u); - code.push_back(EncodeExp0(0x00, 0xf)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeVintrp(0x02, 0, 0, 0, 2)); + AppendVMovLiteral(&code, 1, 0x00000000u); + AppendVMovLiteral(&code, 2, 0x00000000u); + AppendVMovLiteral(&code, 3, 0x3f800000u); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + AppendEnd(&code); - GraphicsCase test; - test.name = "GraphicsFlatInterpolatorExport"; - test.fragment_code = code; - test.expected_pixel = {0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u}; - test.opcodes = {O::VInterpMovF32, O::VMovB32, O::Exp, O::SEndpgm}; - test.pixel_interpolator_settings = {0x00000400u}; - test.vertices = { - 0xbf800000u, 0xbf800000u, 0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u, - 0x40400000u, 0xbf800000u, 0x3f400000u, 0x00000000u, 0x00000000u, 0x3f800000u, - 0xbf800000u, 0x40400000u, 0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u, - }; - return test; + GraphicsCase test; + test.name = "GraphicsFlatInterpolatorExport"; + test.fragment_code = code; + test.expected_pixel = {0x3e800000u, 0x00000000u, 0x00000000u, 0x3f800000u}; + test.opcodes = {O::VInterpMovF32, O::VMovB32, O::Exp, O::SEndpgm}; + test.pixel_interpolator_settings = {0x00000400u}; + test.vertices = { + 0xbf800000u, 0xbf800000u, 0x3e800000u, 0x00000000u, 0x00000000u, + 0x3f800000u, 0x40400000u, 0xbf800000u, 0x3f400000u, 0x00000000u, + 0x00000000u, 0x3f800000u, 0xbf800000u, 0x40400000u, 0x3e800000u, + 0x00000000u, 0x00000000u, 0x3f800000u, + }; + return test; } GraphicsCase GraphicsDsAddtidScratchExport() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSMovB32(124, InlineU32(0))); - AppendVMovLiteral(&code, 0, 0x3f000000u); - code.push_back(EncodeDs0(0xb0, 0)); - code.push_back(EncodeDs1(0, 0, 0)); - code.push_back(EncodeDs0(0xb1, 0)); - code.push_back(EncodeDs1(4, 0, 0)); - code.push_back(EncodeExp0(0x00, 0xf)); - code.push_back(EncodeExp1(4, 4, 4, 4)); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSMovB32(124, InlineU32(0))); + AppendVMovLiteral(&code, 0, 0x3f000000u); + code.push_back(EncodeDs0(0xb0, 0)); + code.push_back(EncodeDs1(0, 0, 0)); + code.push_back(EncodeDs0(0xb1, 0)); + code.push_back(EncodeDs1(4, 0, 0)); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(4, 4, 4, 4)); + AppendEnd(&code); - return {"GraphicsDsAddtidScratchExport", - code, - {0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u}, - {O::SMovB32, O::VMovB32, O::DsWriteAddtidB32, O::DsReadAddtidB32, O::Exp, O::SEndpgm}}; + return {"GraphicsDsAddtidScratchExport", + code, + {0x3f000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u}, + {O::SMovB32, O::VMovB32, O::DsWriteAddtidB32, O::DsReadAddtidB32, + O::Exp, O::SEndpgm}}; } GraphicsCase GraphicsDirectSgprPushConstantExport() { - using O = ShaderOpcode; + using O = ShaderOpcode; - const std::vector values = {0x3e800000u, 0x3f000000u, 0x3f400000u, 0x3f800000u}; + const std::vector values = {0x3e800000u, 0x3f000000u, 0x3f400000u, + 0x3f800000u}; - std::vector code; - code.push_back(EncodeVop1(0x01, 0, 0)); - code.push_back(EncodeVop1(0x01, 1, 1)); - code.push_back(EncodeVop1(0x01, 2, 2)); - code.push_back(EncodeVop1(0x01, 3, 3)); - code.push_back(EncodeExp0(0x00, 0xf)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeVop1(0x01, 0, 0)); + code.push_back(EncodeVop1(0x01, 1, 1)); + code.push_back(EncodeVop1(0x01, 2, 2)); + code.push_back(EncodeVop1(0x01, 3, 3)); + code.push_back(EncodeExp0(0x00, 0xf)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + AppendEnd(&code); - GraphicsCase test; - test.name = "GraphicsDirectSgprPushConstantExport"; - test.fragment_code = code; - test.expected_pixel = values; - test.opcodes = {O::VMovB32, O::Exp, O::SEndpgm}; + GraphicsCase test; + test.name = "GraphicsDirectSgprPushConstantExport"; + test.fragment_code = code; + test.expected_pixel = values; + test.opcodes = {O::VMovB32, O::Exp, O::SEndpgm}; - for (size_t i = 0; i < values.size(); i++) { - test.user_data[i] = values[i]; - } - test.has_user_data = true; - test.push_constants = values; - return test; + for (size_t i = 0; i < values.size(); i++) { + test.user_data[i] = values[i]; + } + test.has_user_data = true; + test.push_constants = values; + return test; } GraphicsCase GraphicsInlineSrtScalarPromotionExport() { - using O = ShaderOpcode; + using O = ShaderOpcode; - const std::vector values = {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u}; + const std::vector values = {0x00000000u, 0x00000000u, 0x00000000u, + 0x3f800000u}; - std::vector code; - AppendVop3(&code, 0x12fu, 0, 0, 1); - AppendVop3(&code, 0x12fu, 1, 2, 3); - code.push_back(EncodeExp0(0x00, 0xf, true, true, true)); - code.push_back(EncodeExp1(0, 1, 0, 0)); - AppendEnd(&code); + std::vector code; + AppendVop3(&code, 0x12fu, 0, 0, 1); + AppendVop3(&code, 0x12fu, 1, 2, 3); + code.push_back(EncodeExp0(0x00, 0xf, true, true, true)); + code.push_back(EncodeExp1(0, 1, 0, 0)); + AppendEnd(&code); - GraphicsCase test; - test.name = "GraphicsInlineSrtScalarPromotionExport"; - test.fragment_code = code; - test.expected_pixel = values; - test.opcodes = {O::VCvtPkrtzF16F32, O::Exp, O::SEndpgm}; + GraphicsCase test; + test.name = "GraphicsInlineSrtScalarPromotionExport"; + test.fragment_code = code; + test.expected_pixel = values; + test.opcodes = {O::VCvtPkrtzF16F32, O::Exp, O::SEndpgm}; - for (size_t i = 0; i < values.size(); i++) { - test.user_data[i] = values[i]; - } - test.has_user_data = true; - test.push_constants = values; - return test; + for (size_t i = 0; i < values.size(); i++) { + test.user_data[i] = values[i]; + } + test.has_user_data = true; + test.push_constants = values; + return test; } GraphicsCase GraphicsNullVmExportDiscardsInactiveExec() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f000000u); - AppendVMovLiteral(&code, 1, 0x3f400000u); - AppendVMovLiteral(&code, 2, 0x3f800000u); - AppendVMovLiteral(&code, 3, 0x40000000u); - code.push_back(EncodeExp0(0x00, 0xf, false)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); - code.push_back(EncodeExp0(0x09, 0x0, true, false, true)); - code.push_back(EncodeExp1(0, 0, 0, 0)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeExp0(0x00, 0xf, false)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x09, 0x0, true, false, true)); + code.push_back(EncodeExp1(0, 0, 0, 0)); + AppendEnd(&code); - return {"GraphicsNullVmExportDiscardsInactiveExec", - code, - {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, - {O::VMovB32, O::Exp, O::SMovB64, O::SEndpgm}}; + return {"GraphicsNullVmExportDiscardsInactiveExec", + code, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {O::VMovB32, O::Exp, O::SMovB64, O::SEndpgm}}; } GraphicsCase GraphicsMrt0OffVmExportDiscardsInactiveExec() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f000000u); - AppendVMovLiteral(&code, 1, 0x3f400000u); - AppendVMovLiteral(&code, 2, 0x3f800000u); - AppendVMovLiteral(&code, 3, 0x40000000u); - code.push_back(EncodeExp0(0x00, 0xf, false)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); - code.push_back(EncodeExp0(0x00, 0x0, true, true, true)); - code.push_back(EncodeExp1(0, 0, 0, 0)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeExp0(0x00, 0xf, false)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x00, 0x0, true, true, true)); + code.push_back(EncodeExp1(0, 0, 0, 0)); + AppendEnd(&code); - return {"GraphicsMrt0OffVmExportDiscardsInactiveExec", - code, - {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, - {O::VMovB32, O::Exp, O::SMovB64, O::SEndpgm}}; + return {"GraphicsMrt0OffVmExportDiscardsInactiveExec", + code, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {O::VMovB32, O::Exp, O::SMovB64, O::SEndpgm}}; } GraphicsCase GraphicsFinalVmExportSupersedesEarlierVmMask() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f000000u); - AppendVMovLiteral(&code, 1, 0x3f400000u); - AppendVMovLiteral(&code, 2, 0x3f800000u); - AppendVMovLiteral(&code, 3, 0x40000000u); - code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); - code.push_back(EncodeExp0(0x00, 0xf, false, false, true)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - code.push_back(EncodeSMovB32(126, 193u)); - code.push_back(EncodeSMovB32(127, 193u)); - AppendVMovLiteral(&code, 4, 0x40400000u); - AppendVMovLiteral(&code, 5, 0x40800000u); - AppendVMovLiteral(&code, 6, 0x40a00000u); - AppendVMovLiteral(&code, 7, 0x40c00000u); - code.push_back(EncodeExp0(0x00, 0xf, true, false, true)); - code.push_back(EncodeExp1(4, 5, 6, 7)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x00, 0xf, false, false, true)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSMovB32(126, 193u)); + code.push_back(EncodeSMovB32(127, 193u)); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 5, 0x40800000u); + AppendVMovLiteral(&code, 6, 0x40a00000u); + AppendVMovLiteral(&code, 7, 0x40c00000u); + code.push_back(EncodeExp0(0x00, 0xf, true, false, true)); + code.push_back(EncodeExp1(4, 5, 6, 7)); + AppendEnd(&code); - return {"GraphicsFinalVmExportSupersedesEarlierVmMask", - code, - {0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u}, - {O::VMovB32, O::SMovB64, O::Exp, O::SMovB32, O::SEndpgm}}; + return {"GraphicsFinalVmExportSupersedesEarlierVmMask", + code, + {0x40400000u, 0x40800000u, 0x40a00000u, 0x40c00000u}, + {O::VMovB32, O::SMovB64, O::Exp, O::SMovB32, O::SEndpgm}}; } GraphicsCase GraphicsBranchPathFinalVmExportDiscardsInactiveExec() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - AppendVMovLiteral(&code, 0, 0x3f000000u); - AppendVMovLiteral(&code, 1, 0x3f400000u); - AppendVMovLiteral(&code, 2, 0x3f800000u); - AppendVMovLiteral(&code, 3, 0x40000000u); - code.push_back(EncodeExp0(0x00, 0xf, false)); - code.push_back(EncodeExp1(0, 1, 2, 3)); - code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); - code.push_back(EncodeExp0(0x09, 0x0, true, false, true)); - code.push_back(EncodeExp1(0, 0, 0, 0)); - code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1))); - const auto branch_index = code.size(); - code.push_back(0); - AppendEnd(&code); - const auto later_export_index = code.size(); - code[branch_index] = EncodeSopp(0x05, static_cast(later_export_index - branch_index - 1u)); - code.push_back(EncodeSMovB32(126, 193u)); - code.push_back(EncodeSMovB32(127, 193u)); - AppendVMovLiteral(&code, 4, 0x40400000u); - AppendVMovLiteral(&code, 5, 0x40800000u); - AppendVMovLiteral(&code, 6, 0x40a00000u); - AppendVMovLiteral(&code, 7, 0x40c00000u); - code.push_back(EncodeExp0(0x00, 0xf, true, false, true)); - code.push_back(EncodeExp1(4, 5, 6, 7)); - AppendEnd(&code); + std::vector code; + AppendVMovLiteral(&code, 0, 0x3f000000u); + AppendVMovLiteral(&code, 1, 0x3f400000u); + AppendVMovLiteral(&code, 2, 0x3f800000u); + AppendVMovLiteral(&code, 3, 0x40000000u); + code.push_back(EncodeExp0(0x00, 0xf, false)); + code.push_back(EncodeExp1(0, 1, 2, 3)); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.push_back(EncodeExp0(0x09, 0x0, true, false, true)); + code.push_back(EncodeExp1(0, 0, 0, 0)); + code.push_back(EncodeSopc(0x06, InlineU32(0), InlineU32(1))); + const auto branch_index = code.size(); + code.push_back(0); + AppendEnd(&code); + const auto later_export_index = code.size(); + code[branch_index] = EncodeSopp( + 0x05, static_cast(later_export_index - branch_index - 1u)); + code.push_back(EncodeSMovB32(126, 193u)); + code.push_back(EncodeSMovB32(127, 193u)); + AppendVMovLiteral(&code, 4, 0x40400000u); + AppendVMovLiteral(&code, 5, 0x40800000u); + AppendVMovLiteral(&code, 6, 0x40a00000u); + AppendVMovLiteral(&code, 7, 0x40c00000u); + code.push_back(EncodeExp0(0x00, 0xf, true, false, true)); + code.push_back(EncodeExp1(4, 5, 6, 7)); + AppendEnd(&code); - return { - "GraphicsBranchPathFinalVmExportDiscardsInactiveExec", - code, - {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, - {O::VMovB32, O::Exp, O::SMovB64, O::SCmpEqU32, O::SCbranchScc1, O::SMovB32, O::SEndpgm}}; + return {"GraphicsBranchPathFinalVmExportDiscardsInactiveExec", + code, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {O::VMovB32, O::Exp, O::SMovB64, O::SCmpEqU32, O::SCbranchScc1, + O::SMovB32, O::SEndpgm}}; } TestCase MultipleWorkitemsGlobalId() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code = { - EncodeVop1(0x01, 1, 0), - EncodeVop2(0x1a, 1, InlineU32(2), 1), - EncodeVop2(0x25, 1, Vgpr(0), 1), - EncodeVop2(0x25, 2, InlineU32(64), 1), - EncodeVop2(0x1a, 3, InlineU32(2), 1), - }; - AppendBufferStoreDword(&code, 2, 3); - AppendEnd(&code); + std::vector code = { + EncodeVop1(0x01, 1, 0), + EncodeVop2(0x1a, 1, InlineU32(2), 1), + EncodeVop2(0x25, 1, Vgpr(0), 1), + EncodeVop2(0x25, 2, InlineU32(64), 1), + EncodeVop2(0x1a, 3, InlineU32(2), 1), + }; + AppendBufferStoreDword(&code, 2, 3); + AppendEnd(&code); - TestCase test; - test.name = "MultipleWorkitemsGlobalId"; - test.code = code; - test.expected = {64, 65, 66, 67, 68, 69, 70, 71}; - test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::BufferStoreDword, O::SEndpgm}; - test.compute_info.threads_num[0] = 4; - test.compute_info.threads_num[1] = 1; - test.compute_info.threads_num[2] = 1; - test.compute_info.group_id[0] = true; - test.compute_info.thread_ids_num = 1; - test.compute_info.workgroup_register = 0; - test.has_compute_info = true; - test.dispatch_x = 2; - return test; + TestCase test; + test.name = "MultipleWorkitemsGlobalId"; + test.code = code; + test.expected = {64, 65, 66, 67, 68, 69, 70, 71}; + test.opcodes = {O::VMovB32, O::VLshlrevB32, O::VAddNcU32, O::BufferStoreDword, + O::SEndpgm}; + test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[1] = 1; + test.compute_info.threads_num[2] = 1; + test.compute_info.group_id[0] = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.workgroup_register = 0; + test.has_compute_info = true; + test.dispatch_x = 2; + return test; } TestCase DispatcherIrreducibleControlFlow() { - using O = ShaderOpcode; + using O = ShaderOpcode; - std::vector code; - code.push_back(EncodeSopp(0x05, 5)); // entry -> B, fallthrough A - AppendVMovU32(&code, 0, 7); - AppendStoreVgpr(&code, 0, 0); - code.push_back(EncodeSopp(0x04, 1)); // A exits while SCC is initially zero - code.push_back(EncodeSopp(0x02, 0xfffau)); // B -> A - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSopp(0x05, 5)); // entry -> B, fallthrough A + AppendVMovU32(&code, 0, 7); + AppendStoreVgpr(&code, 0, 0); + code.push_back(EncodeSopp(0x04, 1)); // A exits while SCC is initially zero + code.push_back(EncodeSopp(0x02, 0xfffau)); // B -> A + AppendEnd(&code); - TestCase test; - test.name = "DispatcherIrreducibleControlFlow"; - test.code = code; - test.expected = {7}; - test.opcodes = {O::SCbranchScc1, O::VMovB32, O::BufferStoreDword, - O::SCbranchScc0, O::SBranch, O::SEndpgm}; - return test; + TestCase test; + test.name = "DispatcherIrreducibleControlFlow"; + test.code = code; + test.expected = {7}; + test.opcodes = {O::SCbranchScc1, O::VMovB32, O::BufferStoreDword, + O::SCbranchScc0, O::SBranch, O::SEndpgm}; + return test; } std::vector MakeCases() { - std::vector cases; - cases.reserve(128); - auto AddCase = [&cases](TestCase (*factory)()) { cases.push_back(factory()); }; + std::vector cases; + cases.reserve(128); + auto AddCase = [&cases](TestCase (*factory)()) { + cases.push_back(factory()); + }; - AddCase(IntegerAddSubMul); - AddCase(BitwiseOps); - AddCase(Shifts); - AddCase(ExactPushConstantExtent); - AddCase(ScalarShiftCountsMaskLowBits); - AddCase(Rdna2ScalarOpcodes); - AddCase(ScalarExtendedArithmetic); - AddCase(ScalarArithmeticSccCarryBorrowOverflow); - AddCase(ScalarMinMaxSccComparisonEdges); - AddCase(ScalarAbsI32UpdatesScc); - AddCase(ScalarShiftLeftAddSccCarryEdges); - AddCase(ScalarCompareOps); - AddCase(ScalarShiftAddAndMaskOps); - AddCase(ScalarNotB64UpdatesScc); - AddCase(ScalarFlbitI32B64Gpu); - AddCase(ScalarSaveExecOps); - AddCase(ScalarOrn2SaveexecUsesSourceOrNotExec); - AddCase(ScalarGetpcWritesNextInstructionPc); - AddCase(ScalarBitfieldPack); - AddCase(ScalarBrevB32PreservesScc); - AddCase(BitfieldExtractWidthPastEndEdges); - AddCase(Scalar64BitOps); - AddCase(ScalarAndn2B64SccBranch); - AddCase(ScalarLiteral); - AddCase(VectorMoves); - AddCase(VectorVop3MoveAppliesFloatSourceModifiers); - AddCase(VectorIntegerOps); - AddCase(Vop2SdwaSubNcExactByte2Destination); - AddCase(Vop2SdwaSubNcPreservesByteAndWordDestinations); - AddCase(Vop2SdwaMinU32PreservesWordDestination); - AddCase(VectorShiftCountsMaskLowBits); - AddCase(VectorVop3IntegerOps); - AddCase(VectorBfeI32ArithmeticShiftMasksField); - AddCase(VectorCarryAndBitCountOps); - AddCase(VectorMbcntUsesThreadMask); - AddCase(VectorAddcWritesPerLaneCarryOut); - AddCase(VectorAddcUsesPerLaneCarryIn); - AddCase(VectorVop3BCarryOutWritesSgprMask); - AddCase(VectorVop3BCarryOutUsesEncodedSdst); - AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310); - AddCase(VectorMadU64U32UnsignedCarryOut); - AddCase(VectorLaneAndPackedOps); - AddCase(CvtPkU8F32PacksSelectedByte); - AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero); - AddCase(PackedMinMaxF16NanAndSignedZeroEdges); - AddCase(VectorMinMaxF16Ops); - AddCase(VectorCvtU16F16Sdwa); - AddCase(VectorMinMaxMed3F16Ops); - AddCase(VectorSpecialF16Ops); - AddCase(VectorWritelaneIgnoresExecMask); - AddCase(VectorReadlaneFromInactiveWrittenLane); - AddCase(VectorLaneWave32RuntimeSelectorWraps); - AddCase(VectorPermlanex16); - AddCase(VectorPermlane16FetchInactiveZero); - AddCase(VectorPermlane16FetchInactiveFi); - AddCase(VectorDppQuadPermuteReverse); - AddCase(VectorDppBankMaskPreservesDestination); - AddCase(VectorDppBoundsControlZeroPreservesDestination); - AddCase(Vop3LdexpSourceModifier); - AddCase(Vop1MoveRelSource); - AddCase(Vop1MoveRelDestination); - AddCase(VectorFloatSpecialOps); - AddCase(MadMixF16LiteralHalfSourceUsesOpsel); - AddCase(MadMixF16NegHiIsAbsAndNegIsIndependent); - AddCase(VectorVop3FmaF16UsesRdna2Opcode34b); - AddCase(VectorFloatArithmeticOps); - AddCase(VectorMinMaxF32NanAndSignedZeroEdges); - AddCase(VectorMed3F32NanUsesMin3Path); - AddCase(VectorFloatConversionOps); - AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange); - AddCase(VectorSpecialF32FlushesDenormalInputs); - AddCase(VectorSinCosMaxFiniteSpecialCases); - AddCase(VectorCompareOps); - AddCase(VectorVop3CompareEqI64OnGpu); - AddCase(VectorVop3CompareNeU64OnGpu); - AddCase(VectorCompareClassF32); - AddCase(VectorCompareF16Ops); - AddCase(Vop2SdwaCndmaskSourceModifier); - AddCase(Vop2SdwaCndmaskFullDestinationWithSubDwordSource); - AddCase(Vop3CndmaskUsesSgprMaskLaneBits); - AddCase(Vop3CndmaskAllowsDataSourceModifier); - AddCase(VectorCompareExecOps); - AddCase(VectorVop3FloatCompareNegSourceModifier); - AddCase(VectorVop3CmpxWritesExecMask); - AddCase(VectorVopcSdwaCmpxWritesExecMask); - AddCase(VectorCompareInvertedMaskSelect); - AddCase(BranchSelect); - AddCase(SimpleLoop); - AddCase(BranchVccnzUsesWholeMask); - AddCase(BranchVccnzUsesCarryProducedWholeMask); - AddCase(ScalarMemoryLoadVariants); - AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); - AddCase(ScalarLoadAlignsComponentsAndMasksAddress); - AddCase(BufferLoadStore); - AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset); - AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset); - AddCase(BufferLoadDwordNoAddressFlagsIgnoresVaddr); - AddCase(BufferLoadDwordIdxenUsesDescriptorStride); - AddCase(BufferStoreDwordIdxenUsesDescriptorStride); - AddCase(BufferStoreDwordAppliesHostOffset); - AddCase(BufferOffsetsUsePackedLaneAndStorageFallback); - AddCase(BufferLoadVariants); - AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress); - AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress); - AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress); - AddCase(BufferStoreVariants); - AddCase(BufferFormatVariants); - AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress); - AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress); - AddCase(BufferFormatStoreVariants); - AddCase(BufferStoreFormatXResource16UintWritesHalfword); - AddCase(BufferLoadFormatXResource8UintZeroExtendsByte); - AddCase(BufferLoadFormatXyResource88UintExtractsBytes); - AddCase(BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents); - AddCase(BufferStoreFormatXyResource88UintWritesBytes); - AddCase(BufferStoreFormatXyzResource3232UintWritesTwoDwords); - AddCase(BufferStoreFormatXyzwResource323232UintWritesThreeDwords); - AddCase(BufferStoreFormatXyResource32UintWritesOneDword); - AddCase(BufferStoreFormatXyzResource8UintWritesOneByte); - AddCase(BufferStoreFormatXAddTidUsesLaneIndex); - AddCase(BufferStoreFormatXDropsOutOfRangeRecord); - AddCase(TBufferLoadVariants); - AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress); - AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress); - AddCase(TBufferLoadFormatX8UintZeroExtendsByte); - AddCase(TBufferLoadFormatX8888UintExtractsFirstByte); - AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride); - AddCase(TBufferLoadFormatX16FloatConvertsToFloat); - AddCase(TBufferStoreFormatX8UintWritesOneByte); - AddCase(TBufferStoreFormatXSintWritesSubDword); - AddCase(TBufferLoadFormatXSintSignExtendsSubDword); - AddCase(TBufferLoadFormatXy1616IntegerComponents); - AddCase(TBufferStoreFormatXy1616IntegerComponents); - AddCase(TBufferLoadFormatXyz16161616UintLoadsHalfwords); - AddCase(TBufferLoadFormatXy1616UnormConvertsToFloat); - AddCase(TBufferLoadFormatXy88UnormConvertsToFloat); - AddCase(TBufferLoadFormatXy88SnormConvertsToFloat); - AddCase(TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents); - AddCase(TBufferLoadFormatXyzw8888UintExtractsBytes); - AddCase(TBufferLoadFormatXyzw1010102SnormConvertsToFloat); - AddCase(TBufferLoadFormatXyz111110FloatUnpacks); - AddCase(TBufferLoadFormatXyzw3232FloatZerosMissingComponents); - AddCase(TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents); - AddCase(TBufferStoreFormatXy88IntegerComponents); - AddCase(TBufferLoadFormatXy88IntegerComponents); - AddCase(TBufferStoreVariants); - AddCase(FlatLoadVariants); - AddCase(FlatVirtualAddressRebasesGuestAllocation); - AddCase(GlobalSignedImmediateRebasesBeforeSaddr); - AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb); - AddCase(FlatStoreVariants); - AddCase(DsReadWriteVariants); - AddCase(DsAppendConsumeUsesEncodedLdsSelector); - AddCase(DsAppendUsesEncodedGdsSelector); - AddCase(DsGdsSubdwordAndAtomicWrites); - AddCase(DsReadWrite2Variants); - AddCase(DsAtomicNoReturnVariants); - AddCase(DsAtomicReturnVariants); - AddCase(DsMiscVariants); - AddCase(DsFloatMinMaxUsesSeparateCompareOperand); - AddCase(DsSwizzleInvalidSourceLaneZero); - AddCase(BufferAtomicVariants); - AddCase(BufferAtomicGlc0DoesNotReturnOldValue); - AddCase(BufferAtomicFMinExactRawGlcModes); - AddCase(BufferAtomicFMinSpecialValues); - AddCase(BufferAtomicFMinContendedWorkgroup); - AddCase(BufferAtomicFMaxExactRawGlcModes); - AddCase(BufferAtomicFMaxSpecialValues); - AddCase(BufferAtomicFMaxContendedWorkgroup); - AddCase(ImageLoadVariants); - AddCase(ImageLoadR32UintUsesIntegerSampledImage); - AddCase(ImageLoad1DUsesScalarCoordinate); - AddCase(ImageLoad1DArrayUsesLayerCoordinate); - AddCase(ImageLoad1DArrayDescriptorUsesSelectedLayer); - AddCase(ImageLoadMipUsesVaddr2Lod2D); - AddCase(ImageLoadMipNsaUsesSelectedAddressVgprs); - AddCase(ImageLoadA16UintCoordsOnGpu); - AddCase(ImageGetResinfoDmaskWidthHeight); - AddCase(ImageGetResinfoDmaskMipLevels); - AddCase(ImageSampleAndGather); - AddCase(ImageSampleA16SamplerCoordsOnGpu); - AddCase(ImageSampleOpcodeAliasUsesNormalCoords); - AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu); - AddCase(ImageSampleA16CompareBiasRdna2AddressOrder); - AddCase(ImageGatherCompareOpcodes); - AddCase(ImageStoreVariants); - AddCase(ImageStoreRgbOneUsesInverseSwizzle); - AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle); - AddCase(ImageStoreBgraUsesInverseSwizzle); - AddCase(ImageStoreYzwxUsesInverseSwizzle); - AddCase(ImageStoreR32FloatUsesFormatlessStorageImage); - AddCase(ImageStoreR32SintUsesRawUintView); - AddCase(ImageStoreR32UintUsesUintStorageImage); - AddCase(ComputeTgSizeSgprUsesWaveMetadata); - AddCase(ImageAtomicVariants); - AddCase(ImageAtomicGlc0DoesNotReturnOldValue); - AddCase(MultipleWorkitemsGlobalId); - AddCase(DispatcherIrreducibleControlFlow); + AddCase(IntegerAddSubMul); + AddCase(BitwiseOps); + AddCase(Shifts); + AddCase(ExactPushConstantExtent); + AddCase(ScalarShiftCountsMaskLowBits); + AddCase(Rdna2ScalarOpcodes); + AddCase(ScalarExtendedArithmetic); + AddCase(ScalarArithmeticSccCarryBorrowOverflow); + AddCase(ScalarMinMaxSccComparisonEdges); + AddCase(ScalarAbsI32UpdatesScc); + AddCase(ScalarShiftLeftAddSccCarryEdges); + AddCase(ScalarCompareOps); + AddCase(ScalarShiftAddAndMaskOps); + AddCase(ScalarNotB64UpdatesScc); + AddCase(ScalarFlbitI32B64Gpu); + AddCase(ScalarSaveExecOps); + AddCase(ScalarOrn2SaveexecUsesSourceOrNotExec); + AddCase(ScalarGetpcWritesNextInstructionPc); + AddCase(ScalarBitfieldPack); + AddCase(ScalarBrevB32PreservesScc); + AddCase(BitfieldExtractWidthPastEndEdges); + AddCase(Scalar64BitOps); + AddCase(ScalarAndn2B64SccBranch); + AddCase(ScalarLiteral); + AddCase(VectorMoves); + AddCase(VectorVop3MoveAppliesFloatSourceModifiers); + AddCase(VectorIntegerOps); + AddCase(Vop2SdwaSubNcExactByte2Destination); + AddCase(Vop2SdwaSubNcPreservesByteAndWordDestinations); + AddCase(Vop2SdwaMinU32PreservesWordDestination); + AddCase(VectorShiftCountsMaskLowBits); + AddCase(VectorVop3IntegerOps); + AddCase(VectorBfeI32ArithmeticShiftMasksField); + AddCase(VectorCarryAndBitCountOps); + AddCase(VectorMbcntUsesThreadMask); + AddCase(VectorAddcWritesPerLaneCarryOut); + AddCase(VectorAddcUsesPerLaneCarryIn); + AddCase(VectorVop3BCarryOutWritesSgprMask); + AddCase(VectorVop3BCarryOutUsesEncodedSdst); + AddCase(VectorVop3BSubCoU32UsesRdna2Opcode310); + AddCase(VectorMadU64U32UnsignedCarryOut); + AddCase(VectorLaneAndPackedOps); + AddCase(CvtPkU8F32PacksSelectedByte); + AddCase(CvtPkrtzF16F32SubnormalRoundsTowardZero); + AddCase(PackedMinMaxF16NanAndSignedZeroEdges); + AddCase(VectorMinMaxF16Ops); + AddCase(VectorCvtU16F16Sdwa); + AddCase(VectorMinMaxMed3F16Ops); + AddCase(VectorSpecialF16Ops); + AddCase(VectorWritelaneIgnoresExecMask); + AddCase(VectorReadlaneFromInactiveWrittenLane); + AddCase(VectorLaneWave32RuntimeSelectorWraps); + AddCase(VectorPermlanex16); + AddCase(VectorPermlane16FetchInactiveZero); + AddCase(VectorPermlane16FetchInactiveFi); + AddCase(VectorDppQuadPermuteReverse); + AddCase(VectorDppBankMaskPreservesDestination); + AddCase(VectorDppBoundsControlZeroPreservesDestination); + AddCase(Vop3LdexpSourceModifier); + AddCase(Vop1MoveRelSource); + AddCase(Vop1MoveRelDestination); + AddCase(VectorFloatSpecialOps); + AddCase(MadMixF16LiteralHalfSourceUsesOpsel); + AddCase(MadMixF16NegHiIsAbsAndNegIsIndependent); + AddCase(VectorVop3FmaF16UsesRdna2Opcode34b); + AddCase(VectorFloatArithmeticOps); + AddCase(VectorMinMaxF32NanAndSignedZeroEdges); + AddCase(VectorMed3F32NanUsesMin3Path); + AddCase(VectorFloatConversionOps); + AddCase(CvtF32ToIntSaturatesNaNAndOutOfRange); + AddCase(VectorSpecialF32FlushesDenormalInputs); + AddCase(VectorRcpIflagF32IntegerReciprocal); + AddCase(VectorSinCosMaxFiniteSpecialCases); + AddCase(VectorCompareOps); + AddCase(VectorVop3CompareEqI64OnGpu); + AddCase(VectorVop3CompareNeU64OnGpu); + AddCase(VectorCompareClassF32); + AddCase(VectorCompareF16Ops); + AddCase(Vop2SdwaCndmaskSourceModifier); + AddCase(Vop2SdwaCndmaskFullDestinationWithSubDwordSource); + AddCase(Vop3CndmaskUsesSgprMaskLaneBits); + AddCase(Vop3CndmaskAllowsDataSourceModifier); + AddCase(VectorCompareExecOps); + AddCase(VectorVop3FloatCompareNegSourceModifier); + AddCase(VectorVop3CmpxWritesExecMask); + AddCase(VectorVopcSdwaCmpxWritesExecMask); + AddCase(VectorCompareInvertedMaskSelect); + AddCase(BranchSelect); + AddCase(SimpleLoop); + AddCase(BranchVccnzUsesWholeMask); + AddCase(BranchVccnzUsesCarryProducedWholeMask); + AddCase(ScalarMemoryLoadVariants); + AddCase(ScalarLoadSignedImmediateOffsetAddsSoffset); + AddCase(ScalarLoadAlignsComponentsAndMasksAddress); + AddCase(BufferLoadStore); + AddCase(BufferLoadDwordOffenIdxenUsesVaddrPlusOneOffset); + AddCase(BufferStoreDwordOffenIdxenUsesVaddrPlusOneOffset); + AddCase(BufferLoadDwordNoAddressFlagsIgnoresVaddr); + AddCase(BufferLoadDwordIdxenUsesDescriptorStride); + AddCase(BufferStoreDwordIdxenUsesDescriptorStride); + AddCase(BufferStoreDwordAppliesHostOffset); + AddCase(BufferOffsetsUsePackedLaneAndStorageFallback); + AddCase(BufferLoadVariants); + AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress); + AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress); + AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress); + AddCase(BufferStoreVariants); + AddCase(BufferFormatVariants); + AddCase(BufferLoadFormatXyzwSnapshotsOverlappingAddress); + AddCase(BufferLoadFormatXyzwInactiveExecPreservesOverlappingAddress); + AddCase(BufferFormatStoreVariants); + AddCase(BufferStoreFormatXResource16UintWritesHalfword); + AddCase(BufferLoadFormatXResource8UintZeroExtendsByte); + AddCase(BufferLoadFormatXyResource88UintExtractsBytes); + AddCase(BufferLoadFormatXyResource8888UnormConvertsFirstTwoComponents); + AddCase(BufferStoreFormatXyResource88UintWritesBytes); + AddCase(BufferStoreFormatXyzResource3232UintWritesTwoDwords); + AddCase(BufferStoreFormatXyzwResource323232UintWritesThreeDwords); + AddCase(BufferStoreFormatXyResource32UintWritesOneDword); + AddCase(BufferStoreFormatXyzResource8UintWritesOneByte); + AddCase(BufferStoreFormatXAddTidUsesLaneIndex); + AddCase(BufferStoreFormatXDropsOutOfRangeRecord); + AddCase(TBufferLoadVariants); + AddCase(TBufferLoadFormatXyzwSnapshotsOverlappingAddress); + AddCase(TBufferLoadFormatXyzwPackedSnapshotsOverlappingAddress); + AddCase(TBufferLoadFormatX8UintZeroExtendsByte); + AddCase(TBufferLoadFormatX8888UintExtractsFirstByte); + AddCase(TBufferLoadFormatXIdxenUsesDescriptorStride); + AddCase(TBufferLoadFormatX16FloatConvertsToFloat); + AddCase(TBufferStoreFormatX8UintWritesOneByte); + AddCase(TBufferStoreFormatXSintWritesSubDword); + AddCase(TBufferLoadFormatXSintSignExtendsSubDword); + AddCase(TBufferLoadFormatXy1616IntegerComponents); + AddCase(TBufferStoreFormatXy1616IntegerComponents); + AddCase(TBufferLoadFormatXyz16161616UintLoadsHalfwords); + AddCase(TBufferLoadFormatXy1616UnormConvertsToFloat); + AddCase(TBufferLoadFormatXy88UnormConvertsToFloat); + AddCase(TBufferLoadFormatXy88SnormConvertsToFloat); + AddCase(TBufferLoadFormatXy8888UnormConvertsFirstTwoComponents); + AddCase(TBufferLoadFormatXyzw8888UintExtractsBytes); + AddCase(TBufferLoadFormatXyzw1010102SnormConvertsToFloat); + AddCase(TBufferLoadFormatXyz111110FloatUnpacks); + AddCase(TBufferLoadFormatXyzw3232FloatZerosMissingComponents); + AddCase(TBufferStoreFormatXyzw3232FloatWritesOnlyPresentComponents); + AddCase(TBufferStoreFormatXy88IntegerComponents); + AddCase(TBufferLoadFormatXy88IntegerComponents); + AddCase(TBufferStoreVariants); + AddCase(FlatLoadVariants); + AddCase(FlatVirtualAddressRebasesGuestAllocation); + AddCase(GlobalSignedImmediateRebasesBeforeSaddr); + AddCase(FlatSegmentIgnoresSaddrAndMasksOffsetMsb); + AddCase(FlatStoreVariants); + AddCase(DsReadWriteVariants); + AddCase(DsAppendConsumeUsesEncodedLdsSelector); + AddCase(DsAppendUsesEncodedGdsSelector); + AddCase(DsGdsSubdwordAndAtomicWrites); + AddCase(DsReadWrite2Variants); + AddCase(DsAtomicNoReturnVariants); + AddCase(DsAtomicReturnVariants); + AddCase(DsMiscVariants); + AddCase(DsFloatMinMaxUsesSeparateCompareOperand); + AddCase(DsSwizzleInvalidSourceLaneZero); + AddCase(BufferAtomicVariants); + AddCase(BufferAtomicGlc0DoesNotReturnOldValue); + AddCase(BufferAtomicFMinExactRawGlcModes); + AddCase(BufferAtomicFMinSpecialValues); + AddCase(BufferAtomicFMinContendedWorkgroup); + AddCase(BufferAtomicFMaxExactRawGlcModes); + AddCase(BufferAtomicFMaxSpecialValues); + AddCase(BufferAtomicFMaxContendedWorkgroup); + AddCase(ImageLoadVariants); + AddCase(ImageLoadR32UintUsesIntegerSampledImage); + AddCase(ImageLoad1DUsesScalarCoordinate); + AddCase(ImageLoad1DArrayUsesLayerCoordinate); + AddCase(ImageLoad1DArrayDescriptorUsesSelectedLayer); + AddCase(ImageLoadMipUsesVaddr2Lod2D); + AddCase(ImageLoadMipNsaUsesSelectedAddressVgprs); + AddCase(ImageLoadA16UintCoordsOnGpu); + AddCase(ImageGetResinfoDmaskWidthHeight); + AddCase(ImageGetResinfoDmaskMipLevels); + AddCase(ImageSampleAndGather); + AddCase(ImageSampleA16SamplerCoordsOnGpu); + AddCase(ImageSampleOpcodeAliasUsesNormalCoords); + AddCase(ImageSampleA16OffsetKeepsTexelOffset32BitOnGpu); + AddCase(ImageSampleA16CompareBiasRdna2AddressOrder); + AddCase(ImageGatherCompareOpcodes); + AddCase(ImageStoreVariants); + AddCase(ImageStoreRgbOneUsesInverseSwizzle); + AddCase(ImageStoreDuplicateSelectorUsesInverseSwizzle); + AddCase(ImageStoreBgraUsesInverseSwizzle); + AddCase(ImageStoreYzwxUsesInverseSwizzle); + AddCase(ImageStoreR32FloatUsesFormatlessStorageImage); + AddCase(ImageStoreR32SintUsesRawUintView); + AddCase(ImageStoreR32UintUsesUintStorageImage); + AddCase(ComputeTgSizeSgprUsesWaveMetadata); + AddCase(ImageAtomicVariants); + AddCase(ImageAtomicGlc0DoesNotReturnOldValue); + AddCase(MultipleWorkitemsGlobalId); + AddCase(DispatcherIrreducibleControlFlow); - return cases; + return cases; } std::vector MakeGraphicsCases() { - return { - GraphicsInterpolationExport(), - GraphicsFlatInterpolatorExport(), - GraphicsDsAddtidScratchExport(), - GraphicsDirectSgprPushConstantExport(), - GraphicsInlineSrtScalarPromotionExport(), - GraphicsNullVmExportDiscardsInactiveExec(), - GraphicsMrt0OffVmExportDiscardsInactiveExec(), - GraphicsFinalVmExportSupersedesEarlierVmMask(), - GraphicsBranchPathFinalVmExportDiscardsInactiveExec(), - }; + return { + GraphicsInterpolationExport(), + GraphicsFlatInterpolatorExport(), + GraphicsDsAddtidScratchExport(), + GraphicsDirectSgprPushConstantExport(), + GraphicsInlineSrtScalarPromotionExport(), + GraphicsNullVmExportDiscardsInactiveExec(), + GraphicsMrt0OffVmExportDiscardsInactiveExec(), + GraphicsFinalVmExportSupersedesEarlierVmMask(), + GraphicsBranchPathFinalVmExportDiscardsInactiveExec(), + }; } std::vector MakeSkippedCases() { - return {ImageStoreMipWritesExplicitMip2D()}; + return {ImageStoreMipWritesExplicitMip2D()}; } void CheckPs5GameExampleImageClearRuntimeShape() { - const auto MakeCode = [] { - std::vector code; - AppendVop3(&code, 0x347u, 4, 8, InlineU32(6), Vgpr(0)); - for (u32 i = 0; i < 4; i++) { - code.push_back(EncodeVop1(0x01u, i, i + 4u)); - } - code.push_back(EncodeMubuf0(0x07u, 0, true, false)); - code.push_back(EncodeMubuf1(0, 0, 4)); - AppendEnd(&code); - return code; - }; - std::array user_data {}; - user_data[0] = 0x00010000u; - user_data[1] = 16u << 16u; - user_data[2] = 64; - user_data[3] = (static_cast(Prospero::BufferFormat::k32_32_32_32UInt) << 12u) | 4u | - (5u << 3u) | (6u << 6u) | (7u << 9u); - std::fill(user_data.begin() + 4, user_data.end(), 0xff000000u); - ShaderComputeInputInfo compute {}; - compute.threads_num[0] = 64; - compute.threads_num[1] = 1; - compute.threads_num[2] = 1; - compute.dispatch_threads_num[0] = 64; - compute.dispatch_threads_num[1] = 1; - compute.dispatch_threads_num[2] = 1; - compute.group_id[0] = true; - compute.dispatch_thread_dimensions = true; - compute.wave_size = 32; - compute.thread_ids_num = 1; - compute.workgroup_register = 8; + const auto MakeCode = [] { + std::vector code; + AppendVop3(&code, 0x347u, 4, 8, InlineU32(6), Vgpr(0)); + for (u32 i = 0; i < 4; i++) { + code.push_back(EncodeVop1(0x01u, i, i + 4u)); + } + code.push_back(EncodeMubuf0(0x07u, 0, true, false)); + code.push_back(EncodeMubuf1(0, 0, 4)); + AppendEnd(&code); + return code; + }; + std::array user_data{}; + user_data[0] = 0x00010000u; + user_data[1] = 16u << 16u; + user_data[2] = 64; + user_data[3] = + (static_cast(Prospero::BufferFormat::k32_32_32_32UInt) << 12u) | + 4u | (5u << 3u) | (6u << 6u) | (7u << 9u); + std::fill(user_data.begin() + 4, user_data.end(), 0xff000000u); + ShaderComputeInputInfo compute{}; + compute.threads_num[0] = 64; + compute.threads_num[1] = 1; + compute.threads_num[2] = 1; + compute.dispatch_threads_num[0] = 64; + compute.dispatch_threads_num[1] = 1; + compute.dispatch_threads_num[2] = 1; + compute.group_id[0] = true; + compute.dispatch_thread_dimensions = true; + compute.wave_size = 32; + compute.thread_ids_num = 1; + compute.workgroup_register = 8; - const auto Compile = [&](const char* stage, const std::vector& code) { - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.wave_size = 32; - options.user_data_base = 0; - options.user_data_count = static_cast(user_data.size()); - options.user_data = user_data.data(); - options.compute_input_info = &compute; - options.dump_ir = false; - ShaderRecompiler::CompileResult result; - std::string error; - Require("Ps5GameExampleImageClear", stage, - ShaderRecompiler::TryRecompile(code, options, result, &error), error); - ValidateSpirv("Ps5GameExampleImageClear", result.spirv); - return result; - }; + const auto Compile = [&](const char *stage, const std::vector &code) { + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.wave_size = 32; + options.user_data_base = 0; + options.user_data_count = static_cast(user_data.size()); + options.user_data = user_data.data(); + options.compute_input_info = &compute; + options.dump_ir = false; + ShaderRecompiler::CompileResult result; + std::string error; + Require("Ps5GameExampleImageClear", stage, + ShaderRecompiler::TryRecompile(code, options, result, &error), + error); + ValidateSpirv("Ps5GameExampleImageClear", result.spirv); + return result; + }; - const auto code = MakeCode(); - auto positive = Compile("exact Prospero kernel", code); + const auto code = MakeCode(); + auto positive = Compile("exact Prospero kernel", code); - compute.stage.program = std::make_shared(positive.program); - compute.stage.resources = - std::make_shared(positive.resources); - ShaderBufferResource descriptor {}; - u32 packed_clear = 0; - uint64_t size = 0; - Require("Ps5GameExampleImageClear", "runtime shape", - ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, descriptor, packed_clear, size) && - descriptor.Base48() == 0x10000u && size == 64u * 16u && packed_clear == 0xff000000u, - "exact Prospero runtime binding did not resolve to a complete clear"); + compute.stage.program = + std::make_shared(positive.program); + compute.stage.resources = + std::make_shared( + positive.resources); + ShaderBufferResource descriptor{}; + u32 packed_clear = 0; + uint64_t size = 0; + Require("Ps5GameExampleImageClear", "runtime shape", + ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, descriptor, + packed_clear, size) && + descriptor.Base48() == 0x10000u && size == 64u * 16u && + packed_clear == 0xff000000u, + "exact Prospero runtime binding did not resolve to a complete clear"); - auto non_repeated = positive.resources; - non_repeated.user_data[7] ^= 1u; - compute.stage.resources = - std::make_shared(non_repeated); - Require("Ps5GameExampleImageClear", "non-repeated clear", - !ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, descriptor, packed_clear, size), - "non-uniform uint4 data was replaced with a color clear"); - compute.stage.resources = - std::make_shared(positive.resources); - compute.dispatch_threads_num[0] = 32; - Require("Ps5GameExampleImageClear", "partial dispatch", - !ResolveComputeImageClear(compute, 32, 1, 1, 0x61u, descriptor, packed_clear, size), - "partial buffer coverage was classified as a complete clear"); - std::printf("[host] %-32s ok\n", "Ps5GameExampleImageClear"); + auto non_repeated = positive.resources; + non_repeated.user_data[7] ^= 1u; + compute.stage.resources = + std::make_shared( + non_repeated); + Require("Ps5GameExampleImageClear", "non-repeated clear", + !ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, descriptor, + packed_clear, size), + "non-uniform uint4 data was replaced with a color clear"); + compute.stage.resources = + std::make_shared( + positive.resources); + compute.dispatch_threads_num[0] = 32; + Require("Ps5GameExampleImageClear", "partial dispatch", + !ResolveComputeImageClear(compute, 32, 1, 1, 0x61u, descriptor, + packed_clear, size), + "partial buffer coverage was classified as a complete clear"); + std::printf("[host] %-32s ok\n", "Ps5GameExampleImageClear"); } void CheckEmbeddedFetchVertexOffset() { - const auto MakeFetch = [](std::initializer_list> adds, - std::optional> late_add = {}, - u32 accumulator_vgpr = 0, bool ngg_sad = false) { - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(0))); - code.push_back(EncodeSmem0(0x02u, 20, 4)); - code.push_back(EncodeSmem1(0)); - const auto AppendOffsets = [&]() { - for (const auto [sgpr, index_vgpr]: adds) { - if (ngg_sad) { - AppendVop3(&code, 0x15du, accumulator_vgpr, sgpr, InlineU32(0), - Vgpr(index_vgpr)); - } else { - AppendVop3B(&code, 0x30fu, accumulator_vgpr, 0, sgpr, Vgpr(index_vgpr)); - } - } - }; - if (ngg_sad) { - AppendOffsets(); - } - code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5)); - if (!ngg_sad) { - AppendOffsets(); - } - code.push_back(EncodeMubuf0(0x03u, 0, true)); - code.push_back(EncodeMubuf1(9, 5, 0)); - if (late_add.has_value()) { - AppendVop3B(&code, 0x30fu, 0, 0, late_add->first, Vgpr(late_add->second)); - } - AppendEnd(&code); - return code; - }; + const auto MakeFetch = [](std::initializer_list> adds, + std::optional> late_add = {}, + u32 accumulator_vgpr = 0, bool ngg_sad = false) { + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSmem0(0x02u, 20, 4)); + code.push_back(EncodeSmem1(0)); + const auto AppendOffsets = [&]() { + for (const auto [sgpr, index_vgpr] : adds) { + if (ngg_sad) { + AppendVop3(&code, 0x15du, accumulator_vgpr, sgpr, InlineU32(0), + Vgpr(index_vgpr)); + } else { + AppendVop3B(&code, 0x30fu, accumulator_vgpr, 0, sgpr, + Vgpr(index_vgpr)); + } + } + }; + if (ngg_sad) { + AppendOffsets(); + } + code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5)); + if (!ngg_sad) { + AppendOffsets(); + } + code.push_back(EncodeMubuf0(0x03u, 0, true)); + code.push_back(EncodeMubuf1(9, 5, 0)); + if (late_add.has_value()) { + AppendVop3B(&code, 0x30fu, 0, 0, late_add->first, Vgpr(late_add->second)); + } + AppendEnd(&code); + return code; + }; - const auto Compile = [&](const char* name, const std::vector& code, u32 slot10) { - std::array user_data {}; - user_data[10] = slot10; - ShaderVertexInputInfo vertex; - vertex.fetch_embedded = true; - vertex.fetch_buffer_reg = 0; - vertex.fetch_attrib_reg = 2; - vertex.resources_num = 1; - vertex.resources_dst[0].attr_id = 0; - vertex.resources_dst[0].registers_num = 4; + const auto Compile = [&](const char *name, const std::vector &code, + u32 slot10) { + std::array user_data{}; + user_data[10] = slot10; + ShaderVertexInputInfo vertex; + vertex.fetch_embedded = true; + vertex.fetch_buffer_reg = 0; + vertex.fetch_attrib_reg = 2; + vertex.resources_num = 1; + vertex.resources_dst[0].attr_id = 0; + vertex.resources_dst[0].registers_num = 4; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.user_data_base = 8; - options.user_data_count = static_cast(user_data.size()); - options.user_data = user_data.data(); - options.vertex_input_info = &vertex; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.user_data_base = 8; + options.user_data_count = static_cast(user_data.size()); + options.user_data = user_data.data(); + options.vertex_input_info = &vertex; - ShaderRecompiler::CompileResult result; - std::string error; - Require(name, "compile", ShaderRecompiler::TryRecompile(code, options, result, &error), - error); - Require(name, "fetch rewrite", vertex.resource_fetch_components[0] == 4, - "encoded fetch sequence was not recognized and rewritten"); - return result; - }; + ShaderRecompiler::CompileResult result; + std::string error; + Require(name, "compile", + ShaderRecompiler::TryRecompile(code, options, result, &error), + error); + Require(name, "fetch rewrite", vertex.resource_fetch_components[0] == 4, + "encoded fetch sequence was not recognized and rewritten"); + return result; + }; - const auto Resolve = [](const ShaderRecompiler::CompileResult& result, u32 index_offset) { - ShaderVertexInputInfo vertex; - vertex.fetch_embedded = true; - vertex.stage.program = - std::make_shared(result.program); - vertex.stage.resources = - std::make_shared(result.resources); - return ResolveVertexOffset(index_offset, vertex); - }; + const auto Resolve = [](const ShaderRecompiler::CompileResult &result, + u32 index_offset) { + ShaderVertexInputInfo vertex; + vertex.fetch_embedded = true; + vertex.stage.program = + std::make_shared(result.program); + vertex.stage.resources = + std::make_shared( + result.resources); + return ResolveVertexOffset(index_offset, vertex); + }; - const auto valid = Compile("EmbeddedFetchVertexOffset", MakeFetch({{18, 0}}), 7); - Require("EmbeddedFetchVertexOffset", "parse", - valid.program.info.vertex_offset_sgpr == 18 && Resolve(valid, 0) == 7 && - Resolve(valid, 5) == 5, - "canonical fetch offset or register index-offset precedence is wrong"); + const auto valid = + Compile("EmbeddedFetchVertexOffset", MakeFetch({{18, 0}}), 7); + Require( + "EmbeddedFetchVertexOffset", "parse", + valid.program.info.vertex_offset_sgpr == 18 && Resolve(valid, 0) == 7 && + Resolve(valid, 5) == 5, + "canonical fetch offset or register index-offset precedence is wrong"); - const auto ngg_code = MakeFetch({{18, 5}}, {}, 5, true); - Require("EmbeddedFetchNggVertexOffset", "encoding", - ngg_code[3] == 0xd55d0005u && ngg_code[4] == 0x04150012u, - "test does not encode the PS5 V_SAD_U32 vertex-offset prolog"); - const auto ngg = Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8); - Require("EmbeddedFetchNggVertexOffset", "parse", - ngg.program.info.vertex_offset_sgpr == 18 && Resolve(ngg, 0) == 8 && - Resolve(ngg, 5) == 5, - "PS5 NGG vertex-index offset or register index-offset precedence is " - "wrong"); + const auto ngg_code = MakeFetch({{18, 5}}, {}, 5, true); + Require("EmbeddedFetchNggVertexOffset", "encoding", + ngg_code[3] == 0xd55d0005u && ngg_code[4] == 0x04150012u, + "test does not encode the PS5 V_SAD_U32 vertex-offset prolog"); + const auto ngg = Compile("EmbeddedFetchNggVertexOffset", ngg_code, 8); + Require("EmbeddedFetchNggVertexOffset", "parse", + ngg.program.info.vertex_offset_sgpr == 18 && Resolve(ngg, 0) == 8 && + Resolve(ngg, 5) == 5, + "PS5 NGG vertex-index offset or register index-offset precedence is " + "wrong"); - const auto pointer = 0x5b7c5100u; - const auto late = - Compile("EmbeddedFetchLateOffset", MakeFetch({}, std::pair {18, 0}), pointer); - const auto conflict = - Compile("EmbeddedFetchConflictingOffset", MakeFetch({{17, 0}, {18, 0}}), pointer); - const auto malformed = Compile("EmbeddedFetchMalformedOffset", MakeFetch({{18, 1}}), pointer); - const auto outside = Compile("EmbeddedFetchOutsideOffset", MakeFetch({{19, 0}}), pointer); - for (const auto* result: {&late, &conflict, &malformed, &outside}) { - Require("EmbeddedFetchVertexOffset", "fail closed", - result->program.info.vertex_offset_sgpr == -1 && Resolve(*result, 0) == 0, - "non-prolog, conflicting, malformed, or out-of-window add was " - "classified"); - } - std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset"); + const auto pointer = 0x5b7c5100u; + const auto late = Compile("EmbeddedFetchLateOffset", + MakeFetch({}, std::pair{18, 0}), pointer); + const auto conflict = Compile("EmbeddedFetchConflictingOffset", + MakeFetch({{17, 0}, {18, 0}}), pointer); + const auto malformed = + Compile("EmbeddedFetchMalformedOffset", MakeFetch({{18, 1}}), pointer); + const auto outside = + Compile("EmbeddedFetchOutsideOffset", MakeFetch({{19, 0}}), pointer); + for (const auto *result : {&late, &conflict, &malformed, &outside}) { + Require("EmbeddedFetchVertexOffset", "fail closed", + result->program.info.vertex_offset_sgpr == -1 && + Resolve(*result, 0) == 0, + "non-prolog, conflicting, malformed, or out-of-window add was " + "classified"); + } + std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset"); } [[noreturn]] void RunReverseRenderTargetDeathCase() { - (void)TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16, - Prospero::ChannelType::kSrgb, - Prospero::ChannelOrder::kStandard); - std::_Exit(0x7f); + (void)TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16, + Prospero::ChannelType::kSrgb, + Prospero::ChannelOrder::kStandard); + std::_Exit(0x7f); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS void CheckRenderTargetFormatContract() { - const auto r8_uint = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k8, - Prospero::ChannelType::kUInt, - Prospero::ChannelOrder::kStandard); - Require("RenderTargetFormat", "R8 uint", - r8_uint.format == vk::Format::eR8Uint && r8_uint.bytes_per_element == 1u && - r8_uint.export_mapping.IsIdentity(), - "R8 uint render-target tuple was rejected"); + const auto r8_uint = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k8, Prospero::ChannelType::kUInt, + Prospero::ChannelOrder::kStandard); + Require("RenderTargetFormat", "R8 uint", + r8_uint.format == vk::Format::eR8Uint && + r8_uint.bytes_per_element == 1u && + r8_uint.export_mapping.IsIdentity(), + "R8 uint render-target tuple was rejected"); - const auto rgb565 = TextureGetRenderTargetFormat(Prospero::ChannelLayout::k5_6_5, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kStandard); - Require("RenderTargetFormat", "RGB565 UNorm", - rgb565.format == vk::Format::eB5G6R5UnormPack16 && rgb565.bytes_per_element == 2u && - rgb565.export_mapping.IsIdentity(), - "RGB565 UNorm render-target tuple was rejected"); + const auto rgb565 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k5_6_5, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kStandard); + Require("RenderTargetFormat", "RGB565 UNorm", + rgb565.format == vk::Format::eB5G6R5UnormPack16 && + rgb565.bytes_per_element == 2u && + rgb565.export_mapping.IsIdentity(), + "RGB565 UNorm render-target tuple was rejected"); - const auto uint_format = TextureGetRenderTargetFormat( - Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kUInt, - Prospero::ChannelOrder::kStandard); - Require("RenderTargetFormat", "RGBA16 uint", - uint_format.format == vk::Format::eR16G16B16A16Uint && - uint_format.bytes_per_element == 8u && uint_format.export_mapping.IsIdentity(), - "RGBA16 uint render-target tuple was rejected"); + const auto uint_format = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kUInt, + Prospero::ChannelOrder::kStandard); + Require("RenderTargetFormat", "RGBA16 uint", + uint_format.format == vk::Format::eR16G16B16A16Uint && + uint_format.bytes_per_element == 8u && + uint_format.export_mapping.IsIdentity(), + "RGBA16 uint render-target tuple was rejected"); - const auto format = TextureGetRenderTargetFormat( - Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, - Prospero::ChannelOrder::kReversed); - Require("ReverseRenderTarget", "exact format", - format.format == vk::Format::eR16G16B16A16Sfloat && format.bytes_per_element == 8u && - format.export_mapping == Prospero::ColorMappingAbgr, - "exact reverse RGBA16F render-target tuple was rejected"); - Require("ReverseRenderTarget", "write masks", - format.export_mapping.ApplyMask(0x1u) == 0x8u && - format.export_mapping.ApplyMask(0x2u) == 0x4u && - format.export_mapping.ApplyMask(0x4u) == 0x2u && - format.export_mapping.ApplyMask(0x8u) == 0x1u && - format.export_mapping.ApplyMask(0xfu) == 0xfu, - "reverse RGBA16F component mask was not mapped exactly once"); + const auto format = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, + Prospero::ChannelOrder::kReversed); + Require("ReverseRenderTarget", "exact format", + format.format == vk::Format::eR16G16B16A16Sfloat && + format.bytes_per_element == 8u && + format.export_mapping == Prospero::ColorMappingAbgr, + "exact reverse RGBA16F render-target tuple was rejected"); + Require("ReverseRenderTarget", "write masks", + format.export_mapping.ApplyMask(0x1u) == 0x8u && + format.export_mapping.ApplyMask(0x2u) == 0x4u && + format.export_mapping.ApplyMask(0x4u) == 0x2u && + format.export_mapping.ApplyMask(0x8u) == 0x1u && + format.export_mapping.ApplyMask(0xfu) == 0xfu, + "reverse RGBA16F component mask was not mapped exactly once"); - const auto rg32 = TextureGetRenderTargetFormat(Prospero::ChannelLayout::k32_32, - Prospero::ChannelType::kFloat, - Prospero::ChannelOrder::kReversed); - Require("ReverseRenderTarget", "GR32 float", - rg32.format == vk::Format::eR32G32Sfloat && rg32.bytes_per_element == 8u && - rg32.export_mapping == Prospero::ColorMappingGr && - rg32.export_mapping.ApplyMask(0x1u) == 0x2u && - rg32.export_mapping.ApplyMask(0x2u) == 0x1u && - rg32.export_mapping.ApplyMask(0x3u) == 0x3u && - rg32.export_mapping.ApplyMask(0xfu) == 0xfu, - "reverse GR32F render-target export or write-mask mapping is incorrect"); + const auto rg32 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kFloat, + Prospero::ChannelOrder::kReversed); + Require( + "ReverseRenderTarget", "GR32 float", + rg32.format == vk::Format::eR32G32Sfloat && + rg32.bytes_per_element == 8u && + rg32.export_mapping == Prospero::ColorMappingGr && + rg32.export_mapping.ApplyMask(0x1u) == 0x2u && + rg32.export_mapping.ApplyMask(0x2u) == 0x1u && + rg32.export_mapping.ApplyMask(0x3u) == 0x3u && + rg32.export_mapping.ApplyMask(0xfu) == 0xfu, + "reverse GR32F render-target export or write-mask mapping is incorrect"); - const auto argb = TextureGetRenderTargetFormat( - Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, - Prospero::ChannelOrder::kAltReversed); - Require("ReverseRenderTarget", "ARGB float", - argb.format == vk::Format::eR16G16B16A16Sfloat && - argb.export_mapping == Prospero::ColorMappingArgb && - argb.export_mapping.ApplyMask(0x1u) == 0x2u && - argb.export_mapping.ApplyMask(0x2u) == 0x4u && - argb.export_mapping.ApplyMask(0x4u) == 0x8u && - argb.export_mapping.ApplyMask(0x8u) == 0x1u, - "alternate-reversed render-target mapping did not invert its ARGB cycle"); + const auto argb = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, + Prospero::ChannelOrder::kAltReversed); + Require( + "ReverseRenderTarget", "ARGB float", + argb.format == vk::Format::eR16G16B16A16Sfloat && + argb.export_mapping == Prospero::ColorMappingArgb && + argb.export_mapping.ApplyMask(0x1u) == 0x2u && + argb.export_mapping.ApplyMask(0x2u) == 0x4u && + argb.export_mapping.ApplyMask(0x4u) == 0x8u && + argb.export_mapping.ApplyMask(0x8u) == 0x1u, + "alternate-reversed render-target mapping did not invert its ARGB cycle"); - char path[MAX_PATH] {}; - Require("ReverseRenderTarget", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - std::string command = std::string("\"") + path + "\" --reverse-rt-death"; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup {sizeof(startup)}; - PROCESS_INFORMATION process {}; - Require("ReverseRenderTarget", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, - CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, - "CreateProcess failed"); - Require("ReverseRenderTarget", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "unsupported adjacent render-target tuple timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("ReverseRenderTarget", "hard failure", exited && exit_code == 321, - "invalid render-target tuple did not retain the fatal guard"); - std::printf("[host] %-32s ok\n", "RenderTargetFormat"); + char path[MAX_PATH]{}; + Require("ReverseRenderTarget", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + std::string command = std::string("\"") + path + "\" --reverse-rt-death"; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("ReverseRenderTarget", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, + FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Require("ReverseRenderTarget", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "unsupported adjacent render-target tuple timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("ReverseRenderTarget", "hard failure", exited && exit_code == 321, + "invalid render-target tuple did not retain the fatal guard"); + std::printf("[host] %-32s ok\n", "RenderTargetFormat"); } #endif -[[noreturn]] void RunImageViewDeathCase(const char* kind) { - if (std::strcmp(kind, "sampled-invalid-selector") == 0) { - (void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 2)); - } else if (std::strcmp(kind, "sampled-incompatible-format") == 0) { - (void)SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR16Unorm, - DstSel(4, 0, 0, 1)); - } else if (std::strcmp(kind, "sampled-invalid-high") == 0) { - (void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(7, 6, 5, 3)); - } else if (std::strcmp(kind, "sampled-depth-format") == 0) { - (void)SelectSampledDepthView(vk::Format::eD24UnormS8Uint, vk::Format::eR32Sfloat, - DstSel(4, 4, 4, 4)); - } else if (std::strcmp(kind, "sampled-depth-swizzle") == 0) { - (void)SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, vk::Format::eR32Sfloat, - DstSel(4, 5, 6, 7)); - } else if (std::strcmp(kind, "storage-incompatible-format") == 0) { - ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb, vk::Format::eR16G16B16A16Unorm, - DstSel(4, 5, 6, 7)); - } else if (std::strcmp(kind, "volume-mip-count") == 0 || - std::strcmp(kind, "volume-slice-range") == 0) { - VulkanHarness vulkan; - auto& graphics = vulkan.RuntimeContext(); - CommandScheduler scheduler(vulkan.RuntimeRenderer(), graphics); - ImageInfo volume_info {}; - volume_info.pixel_format = vk::Format::eR8G8B8A8Unorm; - volume_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; - volume_info.type = Prospero::ImageType::kColor3D; - volume_info.extent = {8, 8, 4}; - volume_info.resources = {2, 1}; - volume_info.pitch = 8; - volume_info.bytes_per_block = 4; - volume_info.samples = 1; - volume_info.tile_mode = Prospero::TileMode::kLinear; - Libs::Graphics::Image volume(graphics, scheduler, volume_info); - ImageViewInfo view {}; - view.format = volume_info.pixel_format; - view.type = vk::ImageViewType::e2DArray; - view.aspect = vk::ImageAspectFlagBits::eColor; - view.base_level = 1; - view.level_count = 1; - view.layer_count = 1; - view.usage = vk::ImageUsageFlagBits::eSampled; - if (std::strcmp(kind, "volume-mip-count") == 0) { - view.base_level = 0; - view.level_count = 2; - } else { - view.base_layer = 1; - view.layer_count = 2; - } - (void)volume.FindView(view); - } else { - ShaderRecompiler::IR::ImageResource resource {}; - resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - resource.written = true; - if (std::strcmp(kind, "storage-kind") == 0) { - resource.kind = ShaderRecompiler::IR::ResourceKind::Image; - } else if (std::strcmp(kind, "storage-no-write") == 0) { - resource.written = false; - } else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) { - resource.atomic = true; - } else if (std::strcmp(kind, "storage-compare") == 0) { - resource.depth_compare = true; - } else if (std::strcmp(kind, "storage-mip") == 0) { - resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; - } else if (std::strcmp(kind, "storage-dimension") == 0) { - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown; - } else { - std::_Exit(0x7e); - } - ValidateStorageImageResource(resource); - } - std::_Exit(0x7f); +[[noreturn]] void RunImageViewDeathCase(const char *kind) { + if (std::strcmp(kind, "sampled-invalid-selector") == 0) { + (void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Unorm, + DstSel(4, 5, 6, 2)); + } else if (std::strcmp(kind, "sampled-incompatible-format") == 0) { + (void)SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR16Unorm, + DstSel(4, 0, 0, 1)); + } else if (std::strcmp(kind, "sampled-invalid-high") == 0) { + (void)SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Unorm, + DstSel(7, 6, 5, 3)); + } else if (std::strcmp(kind, "sampled-depth-format") == 0) { + (void)SelectSampledDepthView(vk::Format::eD24UnormS8Uint, + vk::Format::eR32Sfloat, DstSel(4, 4, 4, 4)); + } else if (std::strcmp(kind, "sampled-depth-swizzle") == 0) { + (void)SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, + vk::Format::eR32Sfloat, DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "storage-incompatible-format") == 0) { + ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb, + vk::Format::eR16G16B16A16Unorm, + DstSel(4, 5, 6, 7)); + } else if (std::strcmp(kind, "volume-mip-count") == 0 || + std::strcmp(kind, "volume-slice-range") == 0) { + VulkanHarness vulkan; + auto &graphics = vulkan.RuntimeContext(); + CommandScheduler scheduler(vulkan.RuntimeRenderer(), graphics); + ImageInfo volume_info{}; + volume_info.pixel_format = vk::Format::eR8G8B8A8Unorm; + volume_info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; + volume_info.type = Prospero::ImageType::kColor3D; + volume_info.extent = {8, 8, 4}; + volume_info.resources = {2, 1}; + volume_info.pitch = 8; + volume_info.bytes_per_block = 4; + volume_info.samples = 1; + volume_info.tile_mode = Prospero::TileMode::kLinear; + Libs::Graphics::Image volume(graphics, scheduler, volume_info); + ImageViewInfo view{}; + view.format = volume_info.pixel_format; + view.type = vk::ImageViewType::e2DArray; + view.aspect = vk::ImageAspectFlagBits::eColor; + view.base_level = 1; + view.level_count = 1; + view.layer_count = 1; + view.usage = vk::ImageUsageFlagBits::eSampled; + if (std::strcmp(kind, "volume-mip-count") == 0) { + view.base_level = 0; + view.level_count = 2; + } else { + view.base_layer = 1; + view.layer_count = 2; + } + (void)volume.FindView(view); + } else { + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.written = true; + if (std::strcmp(kind, "storage-kind") == 0) { + resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + } else if (std::strcmp(kind, "storage-no-write") == 0) { + resource.written = false; + } else if (std::strcmp(kind, "storage-nonuint-atomic") == 0) { + resource.atomic = true; + } else if (std::strcmp(kind, "storage-compare") == 0) { + resource.depth_compare = true; + } else if (std::strcmp(kind, "storage-mip") == 0) { + resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; + } else if (std::strcmp(kind, "storage-dimension") == 0) { + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Unknown; + } else { + std::_Exit(0x7e); + } + ValidateStorageImageResource(resource); + } + std::_Exit(0x7f); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS void CheckSampledColorViews() { - Require("SampledColorViews", "identity", - SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7), - "RGBA did not select the identity view"); - ShaderRecompiler::IR::ImageResource cube_resource {}; - cube_resource.kind = ShaderRecompiler::IR::ResourceKind::Image; - cube_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - cube_resource.read = true; - const auto cube_view = - ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 0, 6); - Require("SampledColorViews", "PPSA17337 cubemap render target", - cube_view.type == vk::ImageViewType::e2DArray && cube_view.base_layer == 0 && - cube_view.layer_count == 6, - "captured six-face cubemap did not resolve to a 2D-array view"); - const auto cube_array_view = - ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 6, 18); - Require("SampledColorViews", "cubemap array subview", - cube_array_view.type == vk::ImageViewType::e2DArray && - cube_array_view.base_layer == 6 && cube_array_view.layer_count == 12, - "nonzero-base multi-cube view did not preserve whole face groups"); - auto non_array_cube_resource = cube_resource; - non_array_cube_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - Require( - "SampledColorViews", "cubemap hard guards", - ResolveTargetTextureView(non_array_cube_resource, Prospero::ImageType::kCube, 0, 6).type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && - ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 0, 7).type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && - ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 1, 6).type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && - ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 6, 6).type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM), - "non-array or partial cubemap views were accepted"); - uint32_t valid_swizzles = 0; - for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) { - bool expected = true; - for (uint32_t channel = 0; channel < 4; channel++) { - const auto selector = GetDstSel(swizzle, channel); - if (selector == 2 || selector == 3) { - expected = false; - } - } - const bool valid = IsValidImageSwizzle(swizzle); - const bool supported = IsSupportedSampledColorView(vk::Format::eR8G8B8A8Unorm, - vk::Format::eR8G8B8A8Unorm, swizzle); - Require("SampledColorViews", "exhaustive read swizzle domain", - valid == expected && supported == expected, - "sampled swizzle validator disagreed with the PS5 selector domain"); - valid_swizzles += valid; - } - Require("SampledColorViews", "all PS5 read swizzles", - valid_swizzles == 1296 && !IsValidImageSwizzle(DstSel(4, 5, 6, 2)) && - !IsValidImageSwizzle(DstSel(4, 5, 6, 3)) && !IsValidImageSwizzle(0x1000), - "valid PS5 sampled mappings were rejected or reserved selectors were " - "admitted"); - const auto arbitrary = DstSel(5, 1, 7, 0); - const auto components = TextureGetComponentMapping(arbitrary); - Require("SampledColorViews", "generic Vulkan component mapping", - SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - arbitrary) == arbitrary && - components.r == vk::ComponentSwizzle::eG && - components.g == vk::ComponentSwizzle::eOne && - components.b == vk::ComponentSwizzle::eA && - components.a == vk::ComponentSwizzle::eZero, - "arbitrary valid sampled mapping did not use the generic view path"); - Require("SampledColorViews", "R8 R001", - SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm, - DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), - "R8 did not select its R001 view"); - Require("SampledColorViews", "R8 000R", - SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm, - DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4), - "R8 did not select its 000R component-mapped view"); - Require("SampledColorViews", "mutable R8 uint/unorm 000R", - SelectSampledColorView(vk::Format::eR8Uint, vk::Format::eR8Unorm, DstSel(0, 0, 0, 4)) == - DstSel(0, 0, 0, 4), - "compatible R8 integer target sampled view was rejected"); - Require("SampledColorViews", "R16G16 RG01", - SelectSampledColorView(vk::Format::eR16G16Sfloat, vk::Format::eR16G16Sfloat, - DstSel(4, 5, 0, 1)) == DstSel(4, 5, 0, 1), - "R16G16 did not select its RG01 view"); - Require("SampledColorViews", "alpha one", - SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 1)) == DstSel(4, 5, 6, 1), - "RGB1 did not select the alpha-one view"); - Require("SampledColorViews", "mutable BGRA target", - SelectSampledColorView(vk::Format::eB8G8R8A8Unorm, vk::Format::eR8G8B8A8Unorm, - DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), - "BGRA target did not select the exact RGBA/BGRA mutable view"); - Require("SampledColorViews", "mutable sRGB view of UNORM BGRA target", - SelectSampledColorView(vk::Format::eB8G8R8A8Unorm, vk::Format::eR8G8B8A8Srgb, - DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), - "UNORM BGRA target did not select its compatible sRGB RGBA sampled view"); - Require("SampledColorViews", "mutable sRGB BGRA target", - SelectSampledColorView(vk::Format::eB8G8R8A8Srgb, vk::Format::eR8G8B8A8Srgb, - DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), - "sRGB BGRA target did not select its matching mutable RGBA view"); - constexpr auto colorspace_swizzle = DstSel(5, 1, 7, 0); - Require("SampledColorViews", "mutable sRGB/UNORM views", - SelectSampledColorView(vk::Format::eR8G8B8A8Srgb, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) && - SelectSampledColorView(vk::Format::eB8G8R8A8Unorm, vk::Format::eB8G8R8A8Srgb, - colorspace_swizzle) == colorspace_swizzle, - "same-order mutable sRGB/UNORM sampled views were rejected"); - Require("SampledColorViews", "mutable packed RGB10 view", - SelectSampledColorView(vk::Format::eA2R10G10B10UnormPack32, - vk::Format::eA2B10G10R10UnormPack32, - DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), - "packed RGB10 target did not select its matching mutable " - "channel-order view"); - Require("SampledColorViews", "reverse RGBA16F sampled view", - SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat, vk::Format::eR16G16B16A16Sfloat, - DstSel(7, 6, 5, 4)) == DstSel(7, 6, 5, 4), - "reverse RGBA16F target did not select its reciprocal ABGR sampled view"); - Require("SampledColorViews", "mutable integer-class views", - SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat, vk::Format::eR16G16B16A16Uint, - DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) && - SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Uint, - DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) && - SelectSampledColorView(vk::Format::eR8G8B8A8Uint, vk::Format::eR8G8B8A8Unorm, - DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7), - "compatible integer render-target sampled view was rejected"); - Require("SampledColorViews", "D32 depth target", - SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, vk::Format::eR32Sfloat, - DstSel(4, 4, 4, 4)) == DstSel(4, 4, 4, 4), - "D32 depth target did not select its depth-aspect view"); - Require("SampledColorViews", "D16 R000 depth target", - SelectSampledDepthView(vk::Format::eD16Unorm, vk::Format::eR16Unorm, - DstSel(4, 0, 0, 0)) == DstSel(4, 0, 0, 0), - "D16 depth target did not select its R000 depth-aspect view"); - Require("SampledColorViews", "D16S8 R001 depth target", - SelectSampledDepthView(vk::Format::eD16UnormS8Uint, vk::Format::eR16Unorm, - DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), - "D16S8 depth target did not select its R001 depth-aspect view"); - Require("SampledColorViews", "promoted D24S8 R001 depth target", - SelectSampledDepthView(vk::Format::eD24UnormS8Uint, vk::Format::eR16Unorm, - DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), - "D24S8 host fallback did not preserve the guest R16 depth view"); - Require("SampledColorViews", "promoted D32S8 R001 depth target", - SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, vk::Format::eR16Unorm, - DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), - "D32S8 host fallback did not preserve the guest R16 depth view"); - Require("SampledColorViews", "D32S8 R001 depth target", - SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, vk::Format::eR32Sfloat, - DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), - "D32S8 depth target did not select its R001 depth-aspect view"); - ShaderRecompiler::IR::ImageResource storage_resource {}; - storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; - storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - storage_resource.written = true; - Require("SampledColorViews", "storage resource", - IsSupportedStorageImageResource(storage_resource), - "exact storage resource contract was rejected"); - storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; - storage_resource.read = true; - Require("SampledColorViews", "read-write 3D storage resource", - IsSupportedStorageImageResource(storage_resource), - "basic read-write 3D storage resource was rejected"); - storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - storage_resource.read = false; - Require("SampledColorViews", "write-only 2D-array storage resource", - IsSupportedStorageImageResource(storage_resource), - "basic write-only 2D-array storage resource was rejected"); - storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - Require("SampledColorViews", "write-only uint 2D-array storage resource", - IsSupportedStorageImageResource(storage_resource), - "basic write-only uint 2D-array storage resource was rejected"); - storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - storage_resource.read = true; - storage_resource.atomic = true; - Require("SampledColorViews", "atomic uint 2D storage resource", - IsSupportedStorageImageResource(storage_resource), - "atomic uint storage resource was rejected"); + Require("SampledColorViews", "identity", + SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Unorm, + DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7), + "RGBA did not select the identity view"); + ShaderRecompiler::IR::ImageResource cube_resource{}; + cube_resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + cube_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + cube_resource.read = true; + const auto cube_view = + ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 0, 6); + Require("SampledColorViews", "PPSA17337 cubemap render target", + cube_view.type == vk::ImageViewType::e2DArray && + cube_view.base_layer == 0 && cube_view.layer_count == 6, + "captured six-face cubemap did not resolve to a 2D-array view"); + const auto cube_array_view = ResolveTargetTextureView( + cube_resource, Prospero::ImageType::kCube, 6, 18); + Require("SampledColorViews", "cubemap array subview", + cube_array_view.type == vk::ImageViewType::e2DArray && + cube_array_view.base_layer == 6 && + cube_array_view.layer_count == 12, + "nonzero-base multi-cube view did not preserve whole face groups"); + auto non_array_cube_resource = cube_resource; + non_array_cube_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + Require("SampledColorViews", "cubemap hard guards", + ResolveTargetTextureView(non_array_cube_resource, + Prospero::ImageType::kCube, 0, 6) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && + ResolveTargetTextureView(cube_resource, + Prospero::ImageType::kCube, 0, 7) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && + ResolveTargetTextureView(cube_resource, + Prospero::ImageType::kCube, 1, 6) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM) && + ResolveTargetTextureView(cube_resource, + Prospero::ImageType::kCube, 6, 6) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM), + "non-array or partial cubemap views were accepted"); + uint32_t valid_swizzles = 0; + for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) { + bool expected = true; + for (uint32_t channel = 0; channel < 4; channel++) { + const auto selector = GetDstSel(swizzle, channel); + if (selector == 2 || selector == 3) { + expected = false; + } + } + const bool valid = IsValidImageSwizzle(swizzle); + const bool supported = IsSupportedSampledColorView( + vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Unorm, swizzle); + Require("SampledColorViews", "exhaustive read swizzle domain", + valid == expected && supported == expected, + "sampled swizzle validator disagreed with the PS5 selector domain"); + valid_swizzles += valid; + } + Require("SampledColorViews", "all PS5 read swizzles", + valid_swizzles == 1296 && !IsValidImageSwizzle(DstSel(4, 5, 6, 2)) && + !IsValidImageSwizzle(DstSel(4, 5, 6, 3)) && + !IsValidImageSwizzle(0x1000), + "valid PS5 sampled mappings were rejected or reserved selectors were " + "admitted"); + const auto arbitrary = DstSel(5, 1, 7, 0); + const auto components = TextureGetComponentMapping(arbitrary); + Require("SampledColorViews", "generic Vulkan component mapping", + SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Unorm, + arbitrary) == arbitrary && + components.r == vk::ComponentSwizzle::eG && + components.g == vk::ComponentSwizzle::eOne && + components.b == vk::ComponentSwizzle::eA && + components.a == vk::ComponentSwizzle::eZero, + "arbitrary valid sampled mapping did not use the generic view path"); + Require("SampledColorViews", "R8 R001", + SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm, + DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), + "R8 did not select its R001 view"); + Require("SampledColorViews", "R8 000R", + SelectSampledColorView(vk::Format::eR8Unorm, vk::Format::eR8Unorm, + DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4), + "R8 did not select its 000R component-mapped view"); + Require("SampledColorViews", "mutable R8 uint/unorm 000R", + SelectSampledColorView(vk::Format::eR8Uint, vk::Format::eR8Unorm, + DstSel(0, 0, 0, 4)) == DstSel(0, 0, 0, 4), + "compatible R8 integer target sampled view was rejected"); + Require("SampledColorViews", "R16G16 RG01", + SelectSampledColorView(vk::Format::eR16G16Sfloat, + vk::Format::eR16G16Sfloat, + DstSel(4, 5, 0, 1)) == DstSel(4, 5, 0, 1), + "R16G16 did not select its RG01 view"); + Require("SampledColorViews", "alpha one", + SelectSampledColorView(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Unorm, + DstSel(4, 5, 6, 1)) == DstSel(4, 5, 6, 1), + "RGB1 did not select the alpha-one view"); + Require("SampledColorViews", "mutable BGRA target", + SelectSampledColorView(vk::Format::eB8G8R8A8Unorm, + vk::Format::eR8G8B8A8Unorm, + DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), + "BGRA target did not select the exact RGBA/BGRA mutable view"); + Require( + "SampledColorViews", "mutable sRGB view of UNORM BGRA target", + SelectSampledColorView(vk::Format::eB8G8R8A8Unorm, + vk::Format::eR8G8B8A8Srgb, + DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), + "UNORM BGRA target did not select its compatible sRGB RGBA sampled view"); + Require("SampledColorViews", "mutable sRGB BGRA target", + SelectSampledColorView(vk::Format::eB8G8R8A8Srgb, + vk::Format::eR8G8B8A8Srgb, + DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), + "sRGB BGRA target did not select its matching mutable RGBA view"); + constexpr auto colorspace_swizzle = DstSel(5, 1, 7, 0); + Require("SampledColorViews", "mutable sRGB/UNORM views", + SelectSampledColorView(vk::Format::eR8G8B8A8Srgb, + vk::Format::eR8G8B8A8Unorm, + DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) && + SelectSampledColorView(vk::Format::eB8G8R8A8Unorm, + vk::Format::eB8G8R8A8Srgb, + colorspace_swizzle) == colorspace_swizzle, + "same-order mutable sRGB/UNORM sampled views were rejected"); + Require("SampledColorViews", "mutable packed RGB10 view", + SelectSampledColorView(vk::Format::eA2R10G10B10UnormPack32, + vk::Format::eA2B10G10R10UnormPack32, + DstSel(6, 5, 4, 7)) == DstSel(6, 5, 4, 7), + "packed RGB10 target did not select its matching mutable " + "channel-order view"); + Require( + "SampledColorViews", "reverse RGBA16F sampled view", + SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat, + vk::Format::eR16G16B16A16Sfloat, + DstSel(7, 6, 5, 4)) == DstSel(7, 6, 5, 4), + "reverse RGBA16F target did not select its reciprocal ABGR sampled view"); + Require("SampledColorViews", "mutable integer-class views", + SelectSampledColorView(vk::Format::eR16G16B16A16Sfloat, + vk::Format::eR16G16B16A16Uint, + DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) && + SelectSampledColorView( + vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Uint, + DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7) && + SelectSampledColorView(vk::Format::eR8G8B8A8Uint, + vk::Format::eR8G8B8A8Unorm, + DstSel(4, 5, 6, 7)) == DstSel(4, 5, 6, 7), + "compatible integer render-target sampled view was rejected"); + Require("SampledColorViews", "D32 depth target", + SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, + vk::Format::eR32Sfloat, + DstSel(4, 4, 4, 4)) == DstSel(4, 4, 4, 4), + "D32 depth target did not select its depth-aspect view"); + Require("SampledColorViews", "D16 R000 depth target", + SelectSampledDepthView(vk::Format::eD16Unorm, vk::Format::eR16Unorm, + DstSel(4, 0, 0, 0)) == DstSel(4, 0, 0, 0), + "D16 depth target did not select its R000 depth-aspect view"); + Require("SampledColorViews", "D16S8 R001 depth target", + SelectSampledDepthView(vk::Format::eD16UnormS8Uint, + vk::Format::eR16Unorm, + DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), + "D16S8 depth target did not select its R001 depth-aspect view"); + Require("SampledColorViews", "promoted D24S8 R001 depth target", + SelectSampledDepthView(vk::Format::eD24UnormS8Uint, + vk::Format::eR16Unorm, + DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), + "D24S8 host fallback did not preserve the guest R16 depth view"); + Require("SampledColorViews", "promoted D32S8 R001 depth target", + SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, + vk::Format::eR16Unorm, + DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), + "D32S8 host fallback did not preserve the guest R16 depth view"); + Require("SampledColorViews", "D32S8 R001 depth target", + SelectSampledDepthView(vk::Format::eD32SfloatS8Uint, + vk::Format::eR32Sfloat, + DstSel(4, 0, 0, 1)) == DstSel(4, 0, 0, 1), + "D32S8 depth target did not select its R001 depth-aspect view"); + ShaderRecompiler::IR::ImageResource storage_resource{}; + storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + storage_resource.written = true; + Require("SampledColorViews", "storage resource", + IsSupportedStorageImageResource(storage_resource), + "exact storage resource contract was rejected"); + storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; + storage_resource.read = true; + Require("SampledColorViews", "read-write 3D storage resource", + IsSupportedStorageImageResource(storage_resource), + "basic read-write 3D storage resource was rejected"); + storage_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + storage_resource.read = false; + Require("SampledColorViews", "write-only 2D-array storage resource", + IsSupportedStorageImageResource(storage_resource), + "basic write-only 2D-array storage resource was rejected"); + storage_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + Require("SampledColorViews", "write-only uint 2D-array storage resource", + IsSupportedStorageImageResource(storage_resource), + "basic write-only uint 2D-array storage resource was rejected"); + storage_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + storage_resource.read = true; + storage_resource.atomic = true; + Require("SampledColorViews", "atomic uint 2D storage resource", + IsSupportedStorageImageResource(storage_resource), + "atomic uint storage resource was rejected"); - char path[MAX_PATH] {}; - Require("SampledColorViews", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - for (const char* kind: - {"sampled-invalid-selector", "sampled-incompatible-format", "sampled-invalid-high", - "sampled-depth-format", "sampled-depth-swizzle", "storage-incompatible-format", - "storage-kind", "storage-no-write", "storage-nonuint-atomic", "storage-compare", - "storage-mip", "storage-dimension", "volume-mip-count", "volume-slice-range"}) { - std::string command = std::string("\"") + path + "\" --image-view-death " + kind; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup {sizeof(startup)}; - PROCESS_INFORMATION process {}; - Require("SampledColorViews", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, - CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, - "CreateProcess failed"); - Require("SampledColorViews", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "unsupported view death case timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("SampledColorViews", "host", exited && exit_code == 321, - std::string(kind) + " component mapping did not report a fatal error"); - } - std::printf("[host] %-32s ok\n", "SampledColorRenderTargetViews"); + char path[MAX_PATH]{}; + Require("SampledColorViews", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : + {"sampled-invalid-selector", "sampled-incompatible-format", + "sampled-invalid-high", "sampled-depth-format", "sampled-depth-swizzle", + "storage-incompatible-format", "storage-kind", "storage-no-write", + "storage-nonuint-atomic", "storage-compare", "storage-mip", + "storage-dimension", "volume-mip-count", "volume-slice-range"}) { + std::string command = + std::string("\"") + path + "\" --image-view-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("SampledColorViews", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, + FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Require("SampledColorViews", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "unsupported view death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("SampledColorViews", "host", exited && exit_code == 321, + std::string(kind) + + " component mapping did not report a fatal error"); + } + std::printf("[host] %-32s ok\n", "SampledColorRenderTargetViews"); } #endif void CheckSampledDepthResource() { - ShaderRecompiler::IR::ImageResource resource {}; - resource.kind = ShaderRecompiler::IR::ResourceKind::Image; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - resource.read = true; - resource.depth_compare = true; - Require("SampledDepthResource", "comparison read", IsSupportedSampledDepthResource(resource), - "basic comparison-sampled depth resource was rejected"); + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = true; + resource.depth_compare = true; + Require("SampledDepthResource", "comparison read", + IsSupportedSampledDepthResource(resource), + "basic comparison-sampled depth resource was rejected"); - resource.depth_compare = false; - Require("SampledDepthResource", "ordinary read", IsSupportedSampledDepthResource(resource), - "basic non-comparison depth read was rejected"); + resource.depth_compare = false; + Require("SampledDepthResource", "ordinary read", + IsSupportedSampledDepthResource(resource), + "basic non-comparison depth read was rejected"); - const auto basic = resource; - resource.read = false; - Require("SampledDepthResource", "read required", !IsSupportedSampledDepthResource(resource), - "non-reading depth resource was accepted"); - resource = basic; - resource.written = true; - Require("SampledDepthResource", "write rejected", !IsSupportedSampledDepthResource(resource), - "writable depth resource was accepted"); - resource = basic; - resource.atomic = true; - Require("SampledDepthResource", "atomic rejected", !IsSupportedSampledDepthResource(resource), - "atomic depth resource was accepted"); - resource = basic; - resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; - Require("SampledDepthResource", "dynamic mip rejected", - !IsSupportedSampledDepthResource(resource), - "dynamic-storage mip depth resource was accepted"); - resource = basic; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - Require("SampledDepthResource", "singleton array accepted", - IsSupportedSampledDepthResource(resource), "array depth resource was rejected"); - const auto singleton_array_view = - ResolveTargetTextureView(resource, Prospero::ImageType::kColor2DArray, 0, 1); - Require("SampledDepthResource", "singleton array view", - singleton_array_view.type == vk::ImageViewType::e2DArray && - singleton_array_view.base_layer == 0 && singleton_array_view.layer_count == 1, - "singleton depth array did not preserve the shader array view type"); - Require("SampledDepthResource", "array type mismatch rejected", - ResolveTargetTextureView(resource, Prospero::ImageType::kColor2D, 0, 1).type == - static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM), - "array shader resource accepted a non-array descriptor view"); - resource = basic; - resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; - Require("SampledDepthResource", "integer rejected", !IsSupportedSampledDepthResource(resource), - "integer depth resource was accepted"); - Require("SampledDepthResource", "integer reinterpret resource", - IsSupportedSampledDepthUintResource(resource), - "read-only uint depth reinterpretation resource was rejected"); - std::printf("[host] %-32s ok\n", "SampledDepthResource"); + const auto basic = resource; + resource.read = false; + Require("SampledDepthResource", "read required", + !IsSupportedSampledDepthResource(resource), + "non-reading depth resource was accepted"); + resource = basic; + resource.written = true; + Require("SampledDepthResource", "write rejected", + !IsSupportedSampledDepthResource(resource), + "writable depth resource was accepted"); + resource = basic; + resource.atomic = true; + Require("SampledDepthResource", "atomic rejected", + !IsSupportedSampledDepthResource(resource), + "atomic depth resource was accepted"); + resource = basic; + resource.mip_mode = ShaderRecompiler::IR::ImageMipMode::DynamicStorage; + Require("SampledDepthResource", "dynamic mip rejected", + !IsSupportedSampledDepthResource(resource), + "dynamic-storage mip depth resource was accepted"); + resource = basic; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + Require("SampledDepthResource", "singleton array accepted", + IsSupportedSampledDepthResource(resource), + "array depth resource was rejected"); + const auto singleton_array_view = ResolveTargetTextureView( + resource, Prospero::ImageType::kColor2DArray, 0, 1); + Require("SampledDepthResource", "singleton array view", + singleton_array_view.type == vk::ImageViewType::e2DArray && + singleton_array_view.base_layer == 0 && + singleton_array_view.layer_count == 1, + "singleton depth array did not preserve the shader array view type"); + Require( + "SampledDepthResource", "array type mismatch rejected", + ResolveTargetTextureView(resource, Prospero::ImageType::kColor2D, 0, 1) + .type == + static_cast(VK_IMAGE_VIEW_TYPE_MAX_ENUM), + "array shader resource accepted a non-array descriptor view"); + resource = basic; + resource.kind = ShaderRecompiler::IR::ResourceKind::ImageUint; + Require("SampledDepthResource", "integer rejected", + !IsSupportedSampledDepthResource(resource), + "integer depth resource was accepted"); + Require("SampledDepthResource", "integer reinterpret resource", + IsSupportedSampledDepthUintResource(resource), + "read-only uint depth reinterpretation resource was rejected"); + std::printf("[host] %-32s ok\n", "SampledDepthResource"); } -void CheckSampledVideoOutView(RenderContext& renderer) { - auto& context = renderer.GetGraphics(); - CommandScheduler scheduler(renderer, context); - ShaderRecompiler::IR::ImageResource resource {}; - resource.kind = ShaderRecompiler::IR::ResourceKind::Image; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - resource.read = true; +void CheckSampledVideoOutView(RenderContext &renderer) { + auto &context = renderer.GetGraphics(); + CommandScheduler scheduler(renderer, context); + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = true; - ShaderTextureResource descriptor {}; - descriptor.fields[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; - ImageInfo info {}; - info.pixel_format = vk::Format::eR8G8B8A8Unorm; - info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; - info.type = Prospero::ImageType::kColor2D; - info.extent = {1, 1, 1}; - info.resources = {1, 1}; - info.pitch = 1; - info.bytes_per_block = 4; - info.samples = 1; - info.tile_mode = Prospero::TileMode::kLinear; - info.mip_layout[0] = {0, 4, 1, 1}; - Image image(context, scheduler, info); - image.usage.video_out = true; - Require("SampledVideoOutView", "basic 2D", - IsSupportedSampledVideoOutView(resource, descriptor, image), - "basic 2D video-out view was rejected"); + ShaderTextureResource descriptor{}; + descriptor.fields[3] = static_cast(Prospero::ImageType::kColor2D) + << 28u; + ImageInfo info{}; + info.pixel_format = vk::Format::eR8G8B8A8Unorm; + info.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; + info.type = Prospero::ImageType::kColor2D; + info.extent = {1, 1, 1}; + info.resources = {1, 1}; + info.pitch = 1; + info.bytes_per_block = 4; + info.samples = 1; + info.tile_mode = Prospero::TileMode::kLinear; + info.mip_layout[0] = {0, 4, 1, 1}; + Image image(context, scheduler, info); + image.usage.video_out = true; + Require("SampledVideoOutView", "basic 2D", + IsSupportedSampledVideoOutView(resource, descriptor, image), + "basic 2D video-out view was rejected"); - const auto basic_resource = resource; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - const bool rejects_array_resource = - !IsSupportedSampledVideoOutView(resource, descriptor, image); - resource = basic_resource; - descriptor.fields[3] = static_cast(Prospero::ImageType::kColor2DArray) << 28u; - const bool rejects_array_descriptor = - !IsSupportedSampledVideoOutView(resource, descriptor, image); - descriptor.fields[3] = static_cast(Prospero::ImageType::kColor2D) << 28u; - descriptor.fields[4] = 1u << 16u; - const bool rejects_base_layer = !IsSupportedSampledVideoOutView(resource, descriptor, image); - descriptor.fields[4] = 1u; - const bool rejects_layer_count = !IsSupportedSampledVideoOutView(resource, descriptor, image); - descriptor.fields[4] = 0; - image.info.resources.layers = 2; - const bool rejects_layered_image = !IsSupportedSampledVideoOutView(resource, descriptor, image); - Require("SampledVideoOutView", "array hard failures", - rejects_array_resource && rejects_array_descriptor && rejects_base_layer && - rejects_layer_count && rejects_layered_image, - "unsupported layered video-out view was accepted"); - std::printf("[host] %-32s ok\n", "SampledVideoOutView"); + const auto basic_resource = resource; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + const bool rejects_array_resource = + !IsSupportedSampledVideoOutView(resource, descriptor, image); + resource = basic_resource; + descriptor.fields[3] = + static_cast(Prospero::ImageType::kColor2DArray) << 28u; + const bool rejects_array_descriptor = + !IsSupportedSampledVideoOutView(resource, descriptor, image); + descriptor.fields[3] = static_cast(Prospero::ImageType::kColor2D) + << 28u; + descriptor.fields[4] = 1u << 16u; + const bool rejects_base_layer = + !IsSupportedSampledVideoOutView(resource, descriptor, image); + descriptor.fields[4] = 1u; + const bool rejects_layer_count = + !IsSupportedSampledVideoOutView(resource, descriptor, image); + descriptor.fields[4] = 0; + image.info.resources.layers = 2; + const bool rejects_layered_image = + !IsSupportedSampledVideoOutView(resource, descriptor, image); + Require("SampledVideoOutView", "array hard failures", + rejects_array_resource && rejects_array_descriptor && + rejects_base_layer && rejects_layer_count && + rejects_layered_image, + "unsupported layered video-out view was accepted"); + std::printf("[host] %-32s ok\n", "SampledVideoOutView"); } -void CheckImageTransitionState(RenderContext& renderer) { - constexpr const char* name = "ImageTransitionState"; - auto& context = renderer.GetGraphics(); - CommandScheduler scheduler(renderer, context); - const auto MakeInfo = [](vk::Format format, uint32_t levels, uint32_t layers) { - ImageInfo info {}; - info.pixel_format = format; - info.guest_format = Prospero::BufferFormat::k8UNorm; - info.type = Prospero::ImageType::kColor2D; - info.extent = {4, 4, 1}; - info.resources = {levels, layers}; - info.pitch = 4; - info.bytes_per_block = 1; - info.samples = 1; - info.tile_mode = Prospero::TileMode::kLinear; - return info; - }; - const auto graphics_stage = - vk::PipelineStageFlagBits2::eAllGraphics | vk::PipelineStageFlagBits2::eComputeShader; - constexpr uint64_t copy_capacity = 128ull << 20; - Require(name, "buffered-copy capacity bands", - ImageTestAccess::CopyRows(32ull << 10, 4096, copy_capacity) == 4096 && - ImageTestAccess::CopyRows(32ull << 10, 4097, copy_capacity) == 4096 && - ImageTestAccess::CopyRows(copy_capacity + 4, 1, copy_capacity) == 0, - "buffered image copy does not split at the fixed scratch capacity"); +void CheckImageTransitionState(RenderContext &renderer) { + constexpr const char *name = "ImageTransitionState"; + auto &context = renderer.GetGraphics(); + CommandScheduler scheduler(renderer, context); + const auto MakeInfo = [](vk::Format format, uint32_t levels, + uint32_t layers) { + ImageInfo info{}; + info.pixel_format = format; + info.guest_format = Prospero::BufferFormat::k8UNorm; + info.type = Prospero::ImageType::kColor2D; + info.extent = {4, 4, 1}; + info.resources = {levels, layers}; + info.pitch = 4; + info.bytes_per_block = 1; + info.samples = 1; + info.tile_mode = Prospero::TileMode::kLinear; + return info; + }; + const auto graphics_stage = vk::PipelineStageFlagBits2::eAllGraphics | + vk::PipelineStageFlagBits2::eComputeShader; + constexpr uint64_t copy_capacity = 128ull << 20; + Require( + name, "buffered-copy capacity bands", + ImageTestAccess::CopyRows(32ull << 10, 4096, copy_capacity) == 4096 && + ImageTestAccess::CopyRows(32ull << 10, 4097, copy_capacity) == 4096 && + ImageTestAccess::CopyRows(copy_capacity + 4, 1, copy_capacity) == 0, + "buffered image copy does not split at the fixed scratch capacity"); - Image image(context, scheduler, MakeInfo(vk::Format::eR8Unorm, 2, 3)); - auto barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); - Require(name, "initial full state", - barriers.size() == 1 && - barriers[0].srcStageMask == vk::PipelineStageFlagBits2::eAllCommands && - barriers[0].srcAccessMask == vk::AccessFlagBits2::eNone && - barriers[0].dstStageMask == graphics_stage && - barriers[0].dstAccessMask == vk::AccessFlagBits2::eShaderRead && - barriers[0].oldLayout == vk::ImageLayout::eUndefined && - barriers[0].newLayout == vk::ImageLayout::eShaderReadOnlyOptimal && - barriers[0].subresourceRange.aspectMask == vk::ImageAspectFlagBits::eColor && - barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS && - barriers[0].subresourceRange.layerCount == VK_REMAINING_ARRAY_LAYERS && - image.backing.state.layout == vk::ImageLayout::eShaderReadOnlyOptimal && - image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead && - image.backing.state.pl_stage == graphics_stage, - "initial transition did not record or retain pinned state"); - Require(name, "repeat read elision", - image - .GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, graphics_stage, {}) - .empty(), - "identical read-only transition emitted a barrier"); - barriers = image.GetBarriers( - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, graphics_stage, {}); - Require(name, "access-only transition", - barriers.size() == 1 && barriers[0].oldLayout == barriers[0].newLayout && - barriers[0].srcAccessMask == vk::AccessFlagBits2::eShaderRead, - "same-layout access change was lost"); + Image image(context, scheduler, MakeInfo(vk::Format::eR8Unorm, 2, 3)); + auto barriers = + image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); + Require( + name, "initial full state", + barriers.size() == 1 && + barriers[0].srcStageMask == + vk::PipelineStageFlagBits2::eAllCommands && + barriers[0].srcAccessMask == vk::AccessFlagBits2::eNone && + barriers[0].dstStageMask == graphics_stage && + barriers[0].dstAccessMask == vk::AccessFlagBits2::eShaderRead && + barriers[0].oldLayout == vk::ImageLayout::eUndefined && + barriers[0].newLayout == vk::ImageLayout::eShaderReadOnlyOptimal && + barriers[0].subresourceRange.aspectMask == + vk::ImageAspectFlagBits::eColor && + barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS && + barriers[0].subresourceRange.layerCount == + VK_REMAINING_ARRAY_LAYERS && + image.backing.state.layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead && + image.backing.state.pl_stage == graphics_stage, + "initial transition did not record or retain pinned state"); + Require(name, "repeat read elision", + image + .GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}) + .empty(), + "identical read-only transition emitted a barrier"); + barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eTransferRead, + graphics_stage, {}); + Require(name, "access-only transition", + barriers.size() == 1 && + barriers[0].oldLayout == barriers[0].newLayout && + barriers[0].srcAccessMask == vk::AccessFlagBits2::eShaderRead, + "same-layout access change was lost"); - const auto CheckRepeatedWrite = [&](vk::AccessFlagBits2 access, const char* label) { - image.backing.state = {}; - image.backing.subresource_states.clear(); - (void)image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {}); - Require(name, label, - image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {}).size() == - 1, - "pinned repeated-write dependency was elided"); - }; - CheckRepeatedWrite(vk::AccessFlagBits2::eTransferWrite, "repeat transfer write"); - CheckRepeatedWrite(vk::AccessFlagBits2::eShaderWrite, "repeat shader write"); - CheckRepeatedWrite(vk::AccessFlagBits2::eMemoryWrite, "repeat memory write"); - image.backing.state = {}; - image.backing.subresource_states.clear(); - const auto attachment_access = - vk::AccessFlagBits2::eColorAttachmentRead | vk::AccessFlagBits2::eColorAttachmentWrite; - (void)image.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal, attachment_access, - graphics_stage, {}); - Require(name, "pinned attachment-write quirk", - image - .GetBarriers(vk::ImageLayout::eColorAttachmentOptimal, attachment_access, - graphics_stage, {}) - .empty(), - "attachment writes diverged from the pinned repeated-write set"); + const auto CheckRepeatedWrite = [&](vk::AccessFlagBits2 access, + const char *label) { + image.backing.state = {}; + image.backing.subresource_states.clear(); + (void)image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, + {}); + Require( + name, label, + image.GetBarriers(vk::ImageLayout::eGeneral, access, graphics_stage, {}) + .size() == 1, + "pinned repeated-write dependency was elided"); + }; + CheckRepeatedWrite(vk::AccessFlagBits2::eTransferWrite, + "repeat transfer write"); + CheckRepeatedWrite(vk::AccessFlagBits2::eShaderWrite, "repeat shader write"); + CheckRepeatedWrite(vk::AccessFlagBits2::eMemoryWrite, "repeat memory write"); + image.backing.state = {}; + image.backing.subresource_states.clear(); + const auto attachment_access = vk::AccessFlagBits2::eColorAttachmentRead | + vk::AccessFlagBits2::eColorAttachmentWrite; + (void)image.GetBarriers(vk::ImageLayout::eColorAttachmentOptimal, + attachment_access, graphics_stage, {}); + Require(name, "pinned attachment-write quirk", + image + .GetBarriers(vk::ImageLayout::eColorAttachmentOptimal, + attachment_access, graphics_stage, {}) + .empty(), + "attachment writes diverged from the pinned repeated-write set"); - image.backing.state = {}; - image.backing.subresource_states.clear(); - (void)image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); - constexpr ImageSubresourceRange mip1_layer0 {1, 1, 0, 1}; - barriers = image.GetBarriers(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite, - graphics_stage, mip1_layer0); - Require( - name, "first partial", - barriers.size() == 1 && barriers[0].subresourceRange.baseMipLevel == 1 && - barriers[0].subresourceRange.baseArrayLayer == 0 && - image.backing.subresource_states.size() == 6 && - image.backing.subresource_states[3].layout == vk::ImageLayout::eGeneral && - image.backing.subresource_states[3].access_mask == vk::AccessFlagBits2::eShaderWrite && - image.backing.subresource_states[1].layout == vk::ImageLayout::eShaderReadOnlyOptimal, - "partial state did not use mip-major indexing"); - constexpr ImageSubresourceRange mip0_layer1 {0, 1, 1, 1}; - barriers = - image.GetBarriers(vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, - vk::PipelineStageFlagBits2::eTransfer, mip0_layer1); - Require(name, "second partial", - barriers.size() == 1 && barriers[0].subresourceRange.baseMipLevel == 0 && - barriers[0].subresourceRange.baseArrayLayer == 1 && - image.backing.subresource_states[1].layout == - vk::ImageLayout::eTransferDstOptimal && - image.backing.subresource_states[1].access_mask == - vk::AccessFlagBits2::eTransferWrite && - image.backing.subresource_states[3].layout == vk::ImageLayout::eGeneral, - "second partial transition changed the wrong cell"); - const ImageSubresourceRange explicit_full {0, 2, 0, 3}; - barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, graphics_stage, explicit_full); - bool saw_mip0_layer1 = false; - bool saw_mip1_layer0 = false; - for (const auto& barrier: barriers) { - saw_mip0_layer1 |= barrier.subresourceRange.baseMipLevel == 0 && - barrier.subresourceRange.baseArrayLayer == 1 && - barrier.oldLayout == vk::ImageLayout::eTransferDstOptimal && - barrier.srcStageMask == vk::PipelineStageFlagBits2::eTransfer; - saw_mip1_layer0 |= barrier.subresourceRange.baseMipLevel == 1 && - barrier.subresourceRange.baseArrayLayer == 0 && - barrier.oldLayout == vk::ImageLayout::eGeneral && - barrier.srcAccessMask == vk::AccessFlagBits2::eShaderWrite; - } - Require(name, "partial normalization", - barriers.size() == 2 && saw_mip0_layer1 && saw_mip1_layer0 && - image.backing.subresource_states.empty() && - image.backing.state.layout == vk::ImageLayout::eShaderReadOnlyOptimal && - image.backing.state.access_mask == vk::AccessFlagBits2::eShaderRead && - image.backing.state.pl_stage == graphics_stage, - "full transition did not reconcile and collapse split state"); - barriers = - image.GetBarriers(vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead, - vk::PipelineStageFlagBits2::eTransfer, {}); - Require(name, "normalized full transition", - barriers.size() == 1 && - barriers[0].subresourceRange.levelCount == VK_REMAINING_MIP_LEVELS && - barriers[0].subresourceRange.layerCount == VK_REMAINING_ARRAY_LAYERS && - barriers[0].srcStageMask == graphics_stage && - barriers[0].dstStageMask == vk::PipelineStageFlagBits2::eTransfer, - "normalized state did not return to one full barrier"); + image.backing.state = {}; + image.backing.subresource_states.clear(); + (void)image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); + constexpr ImageSubresourceRange mip1_layer0{1, 1, 0, 1}; + barriers = image.GetBarriers(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderWrite, + graphics_stage, mip1_layer0); + Require(name, "first partial", + barriers.size() == 1 && + barriers[0].subresourceRange.baseMipLevel == 1 && + barriers[0].subresourceRange.baseArrayLayer == 0 && + image.backing.subresource_states.size() == 6 && + image.backing.subresource_states[3].layout == + vk::ImageLayout::eGeneral && + image.backing.subresource_states[3].access_mask == + vk::AccessFlagBits2::eShaderWrite && + image.backing.subresource_states[1].layout == + vk::ImageLayout::eShaderReadOnlyOptimal, + "partial state did not use mip-major indexing"); + constexpr ImageSubresourceRange mip0_layer1{0, 1, 1, 1}; + barriers = image.GetBarriers( + vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite, + vk::PipelineStageFlagBits2::eTransfer, mip0_layer1); + Require(name, "second partial", + barriers.size() == 1 && + barriers[0].subresourceRange.baseMipLevel == 0 && + barriers[0].subresourceRange.baseArrayLayer == 1 && + image.backing.subresource_states[1].layout == + vk::ImageLayout::eTransferDstOptimal && + image.backing.subresource_states[1].access_mask == + vk::AccessFlagBits2::eTransferWrite && + image.backing.subresource_states[3].layout == + vk::ImageLayout::eGeneral, + "second partial transition changed the wrong cell"); + const ImageSubresourceRange explicit_full{0, 2, 0, 3}; + barriers = image.GetBarriers(vk::ImageLayout::eShaderReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, + explicit_full); + bool saw_mip0_layer1 = false; + bool saw_mip1_layer0 = false; + for (const auto &barrier : barriers) { + saw_mip0_layer1 |= + barrier.subresourceRange.baseMipLevel == 0 && + barrier.subresourceRange.baseArrayLayer == 1 && + barrier.oldLayout == vk::ImageLayout::eTransferDstOptimal && + barrier.srcStageMask == vk::PipelineStageFlagBits2::eTransfer; + saw_mip1_layer0 |= + barrier.subresourceRange.baseMipLevel == 1 && + barrier.subresourceRange.baseArrayLayer == 0 && + barrier.oldLayout == vk::ImageLayout::eGeneral && + barrier.srcAccessMask == vk::AccessFlagBits2::eShaderWrite; + } + Require(name, "partial normalization", + barriers.size() == 2 && saw_mip0_layer1 && saw_mip1_layer0 && + image.backing.subresource_states.empty() && + image.backing.state.layout == + vk::ImageLayout::eShaderReadOnlyOptimal && + image.backing.state.access_mask == + vk::AccessFlagBits2::eShaderRead && + image.backing.state.pl_stage == graphics_stage, + "full transition did not reconcile and collapse split state"); + barriers = image.GetBarriers(vk::ImageLayout::eTransferSrcOptimal, + vk::AccessFlagBits2::eTransferRead, + vk::PipelineStageFlagBits2::eTransfer, {}); + Require(name, "normalized full transition", + barriers.size() == 1 && + barriers[0].subresourceRange.levelCount == + VK_REMAINING_MIP_LEVELS && + barriers[0].subresourceRange.layerCount == + VK_REMAINING_ARRAY_LAYERS && + barriers[0].srcStageMask == graphics_stage && + barriers[0].dstStageMask == vk::PipelineStageFlagBits2::eTransfer, + "normalized state did not return to one full barrier"); - auto depth_info = MakeInfo(vk::Format::eD32SfloatS8Uint, 1, 1); - depth_info.guest_format = Prospero::BufferFormat::k32Float; - depth_info.bytes_per_block = 4; - depth_info.tile_mode = Prospero::TileMode::kDepth; - Image depth(context, scheduler, depth_info); - barriers = depth.GetBarriers(vk::ImageLayout::eDepthStencilReadOnlyOptimal, - vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); - Require(name, "depth stencil aspects", - barriers.size() == 1 && - barriers[0].subresourceRange.aspectMask == - (vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil), - "depth/stencil transition did not use the full image aspect"); + auto depth_info = MakeInfo(vk::Format::eD32SfloatS8Uint, 1, 1); + depth_info.guest_format = Prospero::BufferFormat::k32Float; + depth_info.bytes_per_block = 4; + depth_info.tile_mode = Prospero::TileMode::kDepth; + Image depth(context, scheduler, depth_info); + barriers = + depth.GetBarriers(vk::ImageLayout::eDepthStencilReadOnlyOptimal, + vk::AccessFlagBits2::eShaderRead, graphics_stage, {}); + Require(name, "depth stencil aspects", + barriers.size() == 1 && barriers[0].subresourceRange.aspectMask == + (vk::ImageAspectFlagBits::eDepth | + vk::ImageAspectFlagBits::eStencil), + "depth/stencil transition did not use the full image aspect"); - HW::Context registers {}; - HW::UserConfig user_config {}; - HW::Shader shaders {}; - scheduler.Begin(registers, user_config, shaders); - image.backing.state = {}; - image.backing.subresource_states.clear(); - image.Transit(vk::ImageLayout::eGeneral, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eTransferRead, {}, - scheduler.Current().Handle()); - Require(name, "mixed access stages", - image.backing.state.pl_stage == - (graphics_stage | vk::PipelineStageFlagBits2::eTransfer), - "mixed shader/transfer access omitted a required pipeline stage"); + HW::Context registers{}; + HW::UserConfig user_config{}; + HW::Shader shaders{}; + scheduler.Begin(registers, user_config, shaders); + image.backing.state = {}; + image.backing.subresource_states.clear(); + image.Transit(vk::ImageLayout::eGeneral, + vk::AccessFlagBits2::eShaderRead | + vk::AccessFlagBits2::eTransferRead, + {}, scheduler.Current().Handle()); + Require(name, "mixed access stages", + image.backing.state.pl_stage == + (graphics_stage | vk::PipelineStageFlagBits2::eTransfer), + "mixed shader/transfer access omitted a required pipeline stage"); - auto source_info = depth_info; - source_info.extent = {2, 2, 1}; - source_info.pitch = 2; - source_info.stencil = {0x1000, 4}; - auto destination_info = depth_info; - destination_info.resources.levels = 2; - destination_info.stencil = {0x2000, 16}; - Image source(context, scheduler, source_info); - Image destination(context, scheduler, destination_info); - StreamBuffer upload(context, scheduler, MemoryUsage::Upload, 4096); - StreamBuffer download(context, scheduler, MemoryUsage::Download, 4096); - const auto [upload_data, upload_offset] = upload.Map(128, 4); - Require(name, "depth/stencil upload map", upload_data != nullptr, - "combined depth/stencil upload allocation failed"); - constexpr std::array expected_depth {0x3e800000u, 0x3f000000u, 0x3f400000u, - 0x3f800000u}; - constexpr std::array expected_stencil {0x12, 0x34, 0x56, 0x78}; - std::memcpy(upload_data, expected_depth.data(), sizeof(expected_depth)); - std::memcpy(upload_data + 64, expected_stencil.data(), sizeof(expected_stencil)); - upload.Commit(); - std::array source_copies {}; - source_copies[0].bufferOffset = upload_offset; - source_copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, 1}; - source_copies[0].imageExtent = {2, 2, 1}; - source_copies[1].bufferOffset = upload_offset + 64; - source_copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0, 1}; - source_copies[1].imageExtent = {2, 2, 1}; - source.Upload(source_copies, upload.Handle(), upload_offset, 128); - destination.CopyMip(source, 1, 0); + auto source_info = depth_info; + source_info.extent = {2, 2, 1}; + source_info.pitch = 2; + source_info.stencil = {0x1000, 4}; + auto destination_info = depth_info; + destination_info.resources.levels = 2; + destination_info.stencil = {0x2000, 16}; + Image source(context, scheduler, source_info); + Image destination(context, scheduler, destination_info); + StreamBuffer upload(context, scheduler, MemoryUsage::Upload, 4096); + StreamBuffer download(context, scheduler, MemoryUsage::Download, 4096); + const auto [upload_data, upload_offset] = upload.Map(128, 4); + Require(name, "depth/stencil upload map", upload_data != nullptr, + "combined depth/stencil upload allocation failed"); + constexpr std::array expected_depth{0x3e800000u, 0x3f000000u, + 0x3f400000u, 0x3f800000u}; + constexpr std::array expected_stencil{0x12, 0x34, 0x56, 0x78}; + std::memcpy(upload_data, expected_depth.data(), sizeof(expected_depth)); + std::memcpy(upload_data + 64, expected_stencil.data(), + sizeof(expected_stencil)); + upload.Commit(); + std::array source_copies{}; + source_copies[0].bufferOffset = upload_offset; + source_copies[0].imageSubresource = {vk::ImageAspectFlagBits::eDepth, 0, 0, + 1}; + source_copies[0].imageExtent = {2, 2, 1}; + source_copies[1].bufferOffset = upload_offset + 64; + source_copies[1].imageSubresource = {vk::ImageAspectFlagBits::eStencil, 0, 0, + 1}; + source_copies[1].imageExtent = {2, 2, 1}; + source.Upload(source_copies, upload.Handle(), upload_offset, 128); + destination.CopyMip(source, 1, 0); - const auto [download_data, download_offset] = download.Map(128, 4); - Require(name, "depth/stencil download map", download_data != nullptr, - "combined depth/stencil download allocation failed"); - download.Commit(); - auto destination_copies = source_copies; - destination_copies[0].bufferOffset = download_offset; - destination_copies[0].imageSubresource.mipLevel = 1; - destination_copies[1].bufferOffset = download_offset + 64; - destination_copies[1].imageSubresource.mipLevel = 1; - destination.Download(destination_copies, download.Handle(), download_offset, 128); + const auto [download_data, download_offset] = download.Map(128, 4); + Require(name, "depth/stencil download map", download_data != nullptr, + "combined depth/stencil download allocation failed"); + download.Commit(); + auto destination_copies = source_copies; + destination_copies[0].bufferOffset = download_offset; + destination_copies[0].imageSubresource.mipLevel = 1; + destination_copies[1].bufferOffset = download_offset + 64; + destination_copies[1].imageSubresource.mipLevel = 1; + destination.Download(destination_copies, download.Handle(), download_offset, + 128); - auto buffered_info = MakeInfo(vk::Format::eR8Unorm, 1, 1); - buffered_info.extent = {8, 4, 1}; - buffered_info.pitch = 8; - buffered_info.mip_layout[0] = {0, 32, 8, 4}; - Image buffered_source(context, scheduler, buffered_info); - Image buffered_destination(context, scheduler, buffered_info); - constexpr std::array buffered_expected { - 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, - 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, - 0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f}; - const auto [buffered_upload_data, buffered_upload_offset] = - upload.Map(buffered_expected.size(), 4); - Require(name, "buffered-copy upload map", buffered_upload_data != nullptr, - "buffered-copy source allocation failed"); - std::memcpy(buffered_upload_data, buffered_expected.data(), buffered_expected.size()); - upload.Commit(); - vk::BufferImageCopy buffered_region {}; - buffered_region.bufferOffset = buffered_upload_offset; - buffered_region.bufferRowLength = buffered_info.pitch; - buffered_region.bufferImageHeight = buffered_info.extent.height; - buffered_region.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1}; - buffered_region.imageExtent = buffered_info.extent; - buffered_source.Upload(std::span {&buffered_region, 1}, upload.Handle(), buffered_upload_offset, - buffered_expected.size()); - Buffer copy_scratch(context, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, 16); - buffered_destination.CopyImageWithBuffer(buffered_source, copy_scratch); - const auto [buffered_download_data, buffered_download_offset] = - download.Map(buffered_expected.size(), 4); - Require(name, "buffered-copy download map", buffered_download_data != nullptr, - "buffered-copy destination allocation failed"); - download.Commit(); - buffered_region.bufferOffset = buffered_download_offset; - buffered_destination.Download(std::span {&buffered_region, 1}, download.Handle(), - buffered_download_offset, buffered_expected.size()); - scheduler.Finish(); - download.Invalidate(download_offset, 128); - download.Invalidate(buffered_download_offset, buffered_expected.size()); - Require( - name, "depth/stencil mip contents", - std::memcmp(download_data, expected_depth.data(), sizeof(expected_depth)) == 0 && - std::memcmp(download_data + 64, expected_stencil.data(), sizeof(expected_stencil)) == 0, - "combined depth/stencil mip copy lost an aspect"); - Require(name, "buffered-copy chunk contents", - std::memcmp(buffered_download_data, buffered_expected.data(), - buffered_expected.size()) == 0, - "real buffered image copy lost a row across its scratch boundary"); + auto buffered_info = MakeInfo(vk::Format::eR8Unorm, 1, 1); + buffered_info.extent = {8, 4, 1}; + buffered_info.pitch = 8; + buffered_info.mip_layout[0] = {0, 32, 8, 4}; + Image buffered_source(context, scheduler, buffered_info); + Image buffered_destination(context, scheduler, buffered_info); + constexpr std::array buffered_expected{ + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, + 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, + 0x76, 0x87, 0x98, 0xa9, 0xba, 0xcb, 0xdc, 0xed, 0xfe, 0x0f}; + const auto [buffered_upload_data, buffered_upload_offset] = + upload.Map(buffered_expected.size(), 4); + Require(name, "buffered-copy upload map", buffered_upload_data != nullptr, + "buffered-copy source allocation failed"); + std::memcpy(buffered_upload_data, buffered_expected.data(), + buffered_expected.size()); + upload.Commit(); + vk::BufferImageCopy buffered_region{}; + buffered_region.bufferOffset = buffered_upload_offset; + buffered_region.bufferRowLength = buffered_info.pitch; + buffered_region.bufferImageHeight = buffered_info.extent.height; + buffered_region.imageSubresource = {vk::ImageAspectFlagBits::eColor, 0, 0, 1}; + buffered_region.imageExtent = buffered_info.extent; + buffered_source.Upload(std::span{&buffered_region, 1}, upload.Handle(), + buffered_upload_offset, buffered_expected.size()); + Buffer copy_scratch(context, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, + 16); + buffered_destination.CopyImageWithBuffer(buffered_source, copy_scratch); + const auto [buffered_download_data, buffered_download_offset] = + download.Map(buffered_expected.size(), 4); + Require(name, "buffered-copy download map", buffered_download_data != nullptr, + "buffered-copy destination allocation failed"); + download.Commit(); + buffered_region.bufferOffset = buffered_download_offset; + buffered_destination.Download(std::span{&buffered_region, 1}, + download.Handle(), buffered_download_offset, + buffered_expected.size()); + scheduler.Finish(); + download.Invalidate(download_offset, 128); + download.Invalidate(buffered_download_offset, buffered_expected.size()); + Require(name, "depth/stencil mip contents", + std::memcmp(download_data, expected_depth.data(), + sizeof(expected_depth)) == 0 && + std::memcmp(download_data + 64, expected_stencil.data(), + sizeof(expected_stencil)) == 0, + "combined depth/stencil mip copy lost an aspect"); + Require(name, "buffered-copy chunk contents", + std::memcmp(buffered_download_data, buffered_expected.data(), + buffered_expected.size()) == 0, + "real buffered image copy lost a row across its scratch boundary"); - std::printf("[host] %-32s ok\n", name); + std::printf("[host] %-32s ok\n", name); } -void CheckSampledDepthDescriptor(RenderContext& renderer) { - auto& context = renderer.GetGraphics(); - CommandScheduler scheduler(renderer, context); - const auto make_info = [](uint32_t width, uint32_t height, uint32_t pitch, uint32_t layers, - vk::Format format, Prospero::ImageType type, uint32_t samples = 1) { - ImageInfo info {}; - info.pixel_format = format; - info.guest_format = Prospero::BufferFormat::k32Float; - info.type = type; - info.extent = {width, height, 1}; - info.resources = {1, layers}; - info.pitch = pitch; - info.bytes_per_block = 4; - info.samples = samples; - info.tile_mode = Prospero::TileMode::kDepth; - info.mip_layout[0] = {0, static_cast(pitch) * height * layers * 4, pitch, height}; - return info; - }; +void CheckSampledDepthDescriptor(RenderContext &renderer) { + auto &context = renderer.GetGraphics(); + CommandScheduler scheduler(renderer, context); + const auto make_info = [](uint32_t width, uint32_t height, uint32_t pitch, + uint32_t layers, vk::Format format, + Prospero::ImageType type, uint32_t samples = 1) { + ImageInfo info{}; + info.pixel_format = format; + info.guest_format = Prospero::BufferFormat::k32Float; + info.type = type; + info.extent = {width, height, 1}; + info.resources = {1, layers}; + info.pitch = pitch; + info.bytes_per_block = 4; + info.samples = samples; + info.tile_mode = Prospero::TileMode::kDepth; + info.mip_layout[0] = {0, static_cast(pitch) * height * layers * 4, + pitch, height}; + return info; + }; - ShaderTextureResource descriptor {{0x00eb0900u, 0xc1600000u, 0x00bcc14fu, 0x91800924u, - 0x00000000u, 0x00700000u, 0x00000000u, 0x00000000u}}; - Image image( - context, scheduler, - make_info(1344, 756, 1408, 1, vk::Format::eD32SfloatS8Uint, Prospero::ImageType::kColor2D)); - image.usage.depth_target = true; - Require("SampledDepthDescriptor", "normalized padded pitch", - descriptor.Width5() + 1u == image.info.extent.width && - descriptor.Height5() + 1u == image.info.extent.height && - IsSupportedDepthTargetDescriptor(descriptor, image), - "normalized depth image rejected a valid padded descriptor"); + ShaderTextureResource descriptor{{0x00eb0900u, 0xc1600000u, 0x00bcc14fu, + 0x91800924u, 0x00000000u, 0x00700000u, + 0x00000000u, 0x00000000u}}; + Image image(context, scheduler, + make_info(1344, 756, 1408, 1, vk::Format::eD32SfloatS8Uint, + Prospero::ImageType::kColor2D)); + image.usage.depth_target = true; + Require("SampledDepthDescriptor", "normalized padded pitch", + descriptor.Width5() + 1u == image.info.extent.width && + descriptor.Height5() + 1u == image.info.extent.height && + IsSupportedDepthTargetDescriptor(descriptor, image), + "normalized depth image rejected a valid padded descriptor"); - const ShaderTextureResource disabled_sampler_tweaks {{ - 0x05135600u, - 0xc1600000u, - 0x010dc1dfu, - 0x91800924u, - 0x00000000u, - 0x00000000u, - 0x00000000u, - 0x00000000u, - }}; - Require("SampledDepthDescriptor", "disabled sampler tweaks", - disabled_sampler_tweaks.PerfMod5() == 0 && - IsSupportedDepthTextureEncoding(disabled_sampler_tweaks, image), - "valid sampler modulation factor zero was rejected"); + const ShaderTextureResource disabled_sampler_tweaks{{ + 0x05135600u, + 0xc1600000u, + 0x010dc1dfu, + 0x91800924u, + 0x00000000u, + 0x00000000u, + 0x00000000u, + 0x00000000u, + }}; + Require("SampledDepthDescriptor", "disabled sampler tweaks", + disabled_sampler_tweaks.PerfMod5() == 0 && + IsSupportedDepthTextureEncoding(disabled_sampler_tweaks, image), + "valid sampler modulation factor zero was rejected"); - const ShaderTextureResource uncompressed_msaa {{ - 0x00705d00u, - 0xc1600000u, - 0x010dc1dfu, - 0xe1810924u, - 0x00000000u, - 0x00700010u, - 0x00000000u, - 0x00000000u, - }}; - auto msaa_info = - make_info(1920, 1080, 1920, 1, vk::Format::eD32Sfloat, Prospero::ImageType::kColor2D, 2); - msaa_info.mip_layout[0] = {0, 0x010e0000, 1920, 1152}; - Image msaa_image(context, scheduler, msaa_info); - msaa_image.usage.depth_target = true; - Require("SampledDepthDescriptor", "uncompressed 2x MSAA depth", - IsSupportedDepthTargetDescriptor(uncompressed_msaa, msaa_image) && - IsSupportedDepthTextureEncoding(uncompressed_msaa, msaa_image), - "valid uncompressed MSAA depth descriptor required an HTILE " - "compatibility flag"); + const ShaderTextureResource uncompressed_msaa{{ + 0x00705d00u, + 0xc1600000u, + 0x010dc1dfu, + 0xe1810924u, + 0x00000000u, + 0x00700010u, + 0x00000000u, + 0x00000000u, + }}; + auto msaa_info = make_info(1920, 1080, 1920, 1, vk::Format::eD32Sfloat, + Prospero::ImageType::kColor2D, 2); + msaa_info.mip_layout[0] = {0, 0x010e0000, 1920, 1152}; + Image msaa_image(context, scheduler, msaa_info); + msaa_image.usage.depth_target = true; + Require("SampledDepthDescriptor", "uncompressed 2x MSAA depth", + IsSupportedDepthTargetDescriptor(uncompressed_msaa, msaa_image) && + IsSupportedDepthTextureEncoding(uncompressed_msaa, msaa_image), + "valid uncompressed MSAA depth descriptor required an HTILE " + "compatibility flag"); - descriptor.fields[3] = (descriptor.fields[3] & ~(0xfu << 28u)) | - (static_cast(Prospero::ImageType::kColor2DArray) << 28u); - Require("SampledDepthDescriptor", "singleton array descriptor", - IsSupportedDepthTargetDescriptor(descriptor, image), - "normalized singleton-array depth view was rejected"); - descriptor.fields[3] = (descriptor.fields[3] & ~(0xfu << 28u)) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); + descriptor.fields[3] = + (descriptor.fields[3] & ~(0xfu << 28u)) | + (static_cast(Prospero::ImageType::kColor2DArray) << 28u); + Require("SampledDepthDescriptor", "singleton array descriptor", + IsSupportedDepthTargetDescriptor(descriptor, image), + "normalized singleton-array depth view was rejected"); + descriptor.fields[3] = + (descriptor.fields[3] & ~(0xfu << 28u)) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); - ShaderTextureResource cube_descriptor {{0x01267d00u, 0xc0700000u, 0x00ffc0ffu, 0xb1800924u, - 0x00000005u, 0x00700000u, 0x00000000u, 0x00000000u}}; - Image cube_image( - context, scheduler, - make_info(1024, 1024, 1024, 6, vk::Format::eD32Sfloat, Prospero::ImageType::kColor2D)); - cube_image.usage.depth_target = true; - ShaderRecompiler::IR::ImageResource cube_resource {}; - cube_resource.kind = ShaderRecompiler::IR::ResourceKind::Image; - cube_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - cube_resource.read = true; - cube_resource.depth_compare = true; - const auto cube_view = ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 0, - cube_image.info.resources.layers); - Require("SampledDepthDescriptor", "normalized depth cube", - IsSupportedDepthTargetDescriptor(cube_descriptor, cube_image) && - IsSupportedDepthTextureEncoding(cube_descriptor, cube_image) && - cube_view.type == vk::ImageViewType::e2DArray && cube_view.layer_count == 6, - "normalized depth cube did not preserve its six-face view"); + ShaderTextureResource cube_descriptor{{0x01267d00u, 0xc0700000u, 0x00ffc0ffu, + 0xb1800924u, 0x00000005u, 0x00700000u, + 0x00000000u, 0x00000000u}}; + Image cube_image(context, scheduler, + make_info(1024, 1024, 1024, 6, vk::Format::eD32Sfloat, + Prospero::ImageType::kColor2D)); + cube_image.usage.depth_target = true; + ShaderRecompiler::IR::ImageResource cube_resource{}; + cube_resource.kind = ShaderRecompiler::IR::ResourceKind::Image; + cube_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + cube_resource.read = true; + cube_resource.depth_compare = true; + const auto cube_view = + ResolveTargetTextureView(cube_resource, Prospero::ImageType::kCube, 0, + cube_image.info.resources.layers); + Require("SampledDepthDescriptor", "normalized depth cube", + IsSupportedDepthTargetDescriptor(cube_descriptor, cube_image) && + IsSupportedDepthTextureEncoding(cube_descriptor, cube_image) && + cube_view.type == vk::ImageViewType::e2DArray && + cube_view.layer_count == 6, + "normalized depth cube did not preserve its six-face view"); - constexpr uint64_t captured_htile_address = 0x106d48000ull; - const ShaderTextureResource compressed_descriptor {{ - 0x0104d500u, - 0xc1600000u, - 0x021bc3bfu, - 0x91800924u, - 0x00000000u, - 0x00700000u, - 0x80280000u, - 0x000106d4u, - }}; - image.info.metadata.range = {captured_htile_address, 0x1000}; - image.info.metadata.kind = ImageMetadataKind::Htile; - auto mismatched_metadata = compressed_descriptor; - mismatched_metadata.fields[7] ^= 1u; - auto mismatched_metadata_low = compressed_descriptor; - mismatched_metadata_low.fields[6] ^= 1u << 24u; - auto dcc_only_control = compressed_descriptor; - dcc_only_control.fields[6] |= 1u << 22u; - const bool accepts_compressed = IsSupportedDepthTextureEncoding(compressed_descriptor, image); - image.info.metadata.kind = ImageMetadataKind::Dcc; - const bool rejects_dcc = !IsSupportedDepthTextureEncoding(compressed_descriptor, image); - image.info.metadata.kind = ImageMetadataKind::None; - const bool rejects_none = !IsSupportedDepthTextureEncoding(compressed_descriptor, image); - image.info.metadata.kind = ImageMetadataKind::Htile; - image.info.metadata.range.size = 0; - const bool rejects_empty = !IsSupportedDepthTextureEncoding(compressed_descriptor, image); - image.info.metadata.range.size = TRACKER_ADDRESS_SIZE - captured_htile_address + 1u; - const bool rejects_overflow = !IsSupportedDepthTextureEncoding(compressed_descriptor, image); - image.info.metadata.range.size = 0x1000; - Require("SampledDepthDescriptor", "compressed HTILE descriptor", - accepts_compressed && !IsSupportedDepthTextureEncoding(mismatched_metadata, image) && - !IsSupportedDepthTextureEncoding(mismatched_metadata_low, image) && - !IsSupportedDepthTextureEncoding(dcc_only_control, image), - "compressed sampled depth did not require its exact tracked HTILE"); - Require("SampledDepthDescriptor", "tracked HTILE state", - rejects_dcc && rejects_none && rejects_empty && rejects_overflow, - "compressed sampled depth accepted invalid tracked metadata"); + constexpr uint64_t captured_htile_address = 0x106d48000ull; + const ShaderTextureResource compressed_descriptor{{ + 0x0104d500u, + 0xc1600000u, + 0x021bc3bfu, + 0x91800924u, + 0x00000000u, + 0x00700000u, + 0x80280000u, + 0x000106d4u, + }}; + image.info.metadata.range = {captured_htile_address, 0x1000}; + image.info.metadata.kind = ImageMetadataKind::Htile; + auto mismatched_metadata = compressed_descriptor; + mismatched_metadata.fields[7] ^= 1u; + auto mismatched_metadata_low = compressed_descriptor; + mismatched_metadata_low.fields[6] ^= 1u << 24u; + auto dcc_only_control = compressed_descriptor; + dcc_only_control.fields[6] |= 1u << 22u; + const bool accepts_compressed = + IsSupportedDepthTextureEncoding(compressed_descriptor, image); + image.info.metadata.kind = ImageMetadataKind::Dcc; + const bool rejects_dcc = + !IsSupportedDepthTextureEncoding(compressed_descriptor, image); + image.info.metadata.kind = ImageMetadataKind::None; + const bool rejects_none = + !IsSupportedDepthTextureEncoding(compressed_descriptor, image); + image.info.metadata.kind = ImageMetadataKind::Htile; + image.info.metadata.range.size = 0; + const bool rejects_empty = + !IsSupportedDepthTextureEncoding(compressed_descriptor, image); + image.info.metadata.range.size = + TRACKER_ADDRESS_SIZE - captured_htile_address + 1u; + const bool rejects_overflow = + !IsSupportedDepthTextureEncoding(compressed_descriptor, image); + image.info.metadata.range.size = 0x1000; + Require( + "SampledDepthDescriptor", "compressed HTILE descriptor", + accepts_compressed && + !IsSupportedDepthTextureEncoding(mismatched_metadata, image) && + !IsSupportedDepthTextureEncoding(mismatched_metadata_low, image) && + !IsSupportedDepthTextureEncoding(dcc_only_control, image), + "compressed sampled depth did not require its exact tracked HTILE"); + Require("SampledDepthDescriptor", "tracked HTILE state", + rejects_dcc && rejects_none && rejects_empty && rejects_overflow, + "compressed sampled depth accepted invalid tracked metadata"); - auto partial_cube = cube_descriptor; - partial_cube.fields[4] = 4; - auto based_cube = cube_descriptor; - based_cube.fields[4] |= 1u << 16u; - auto reserved_cube = cube_descriptor; - reserved_cube.fields[4] |= 1u << 13u; - auto non_square_cube = cube_descriptor; - non_square_cube.fields[2] = (non_square_cube.fields[2] & ~(0x3fffu << 14u)) | (511u << 14u); - image.info.pitch = 1344; - const bool rejects_pitch = !IsSupportedDepthTargetDescriptor(descriptor, image); - image.info.pitch = 1408; - Require("SampledDepthDescriptor", "normalized hard guards", - rejects_pitch && !IsSupportedDepthTargetDescriptor(partial_cube, cube_image) && - !IsSupportedDepthTargetDescriptor(based_cube, cube_image) && - !IsSupportedDepthTextureEncoding(reserved_cube, cube_image) && - !IsSupportedDepthTargetDescriptor(non_square_cube, cube_image), - "normalized depth descriptor accepted an incompatible image view"); - std::printf("[host] %-32s ok\n", "SampledDepthDescriptor"); + auto partial_cube = cube_descriptor; + partial_cube.fields[4] = 4; + auto based_cube = cube_descriptor; + based_cube.fields[4] |= 1u << 16u; + auto reserved_cube = cube_descriptor; + reserved_cube.fields[4] |= 1u << 13u; + auto non_square_cube = cube_descriptor; + non_square_cube.fields[2] = + (non_square_cube.fields[2] & ~(0x3fffu << 14u)) | (511u << 14u); + image.info.pitch = 1344; + const bool rejects_pitch = + !IsSupportedDepthTargetDescriptor(descriptor, image); + image.info.pitch = 1408; + Require("SampledDepthDescriptor", "normalized hard guards", + rejects_pitch && + !IsSupportedDepthTargetDescriptor(partial_cube, cube_image) && + !IsSupportedDepthTargetDescriptor(based_cube, cube_image) && + !IsSupportedDepthTextureEncoding(reserved_cube, cube_image) && + !IsSupportedDepthTargetDescriptor(non_square_cube, cube_image), + "normalized depth descriptor accepted an incompatible image view"); + std::printf("[host] %-32s ok\n", "SampledDepthDescriptor"); } ShaderRecompiler::IR::ImageResource BasicStorageTextureResource() { - ShaderRecompiler::IR::ImageResource resource {}; - resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; - resource.read = true; - resource.written = true; - return resource; + ShaderRecompiler::IR::ImageResource resource{}; + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim3D; + resource.read = true; + resource.written = true; + return resource; } ShaderTextureResource BasicStorageTextureDescriptor() { - return {{0x00785d00u, 0x04700000u, 0x00080008u, 0xa1b00facu, 0x00000020u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x00785d00u, 0x04700000u, 0x00080008u, 0xa1b00facu, 0x00000020u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource BasicLinearStorageTextureResource() { - auto resource = BasicStorageTextureResource(); - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - resource.read = false; - return resource; + auto resource = BasicStorageTextureResource(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = false; + return resource; } ShaderTextureResource BasicLinearStorageTextureDescriptor() { - return {{0x04bcc401u, 0xc3800000u, 0x021bc3bfu, 0x900003acu, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x04bcc401u, 0xc3800000u, 0x021bc3bfu, 0x900003acu, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource BasicBgraStorageTextureResource() { - auto resource = BasicStorageTextureResource(); - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - resource.read = false; - return resource; + auto resource = BasicStorageTextureResource(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + resource.read = false; + return resource; } ShaderTextureResource BasicBgraStorageTextureDescriptor() { - return {{0x007c6500u, 0xc3800000u, 0x010dc1dfu, 0x91b00f2eu, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x007c6500u, 0xc3800000u, 0x010dc1dfu, 0x91b00f2eu, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Ppsa06228R11G11B10StorageTextureDescriptor() { - return {{0x10c6b500u, 0xc2400000u, 0x010dc1dfu, 0x91b003acu, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x10c6b500u, 0xc2400000u, 0x010dc1dfu, 0x91b003acu, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Ppsa01530MaxMipStorageTextureDescriptor() { - return {{0x04a42900u, 0xc3e00000u, 0x000fc00fu, 0x91b0022cu, 0x00000000u, 0x00700050u, - 0x00000000u, 0x00000000u}}; + return {{0x04a42900u, 0xc3e00000u, 0x000fc00fu, 0x91b0022cu, 0x00000000u, + 0x00700050u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource Ppsa01530MaxMipStorageTextureResource() { - auto resource = BasicBgraStorageTextureResource(); - resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - return resource; + auto resource = BasicBgraStorageTextureResource(); + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + return resource; } ShaderTextureResource Ppsa02527R16FloatStorageTextureDescriptor() { - return {{0x00ce3500u, 0xc0d00000u, 0x010dc1dfu, 0x91b00204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x00ce3500u, 0xc0d00000u, 0x010dc1dfu, 0x91b00204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Ppsa02527R32FloatStorageTextureDescriptor() { - return {{0x00cea900u, 0xc1600000u, 0x0086c0efu, 0x91b00204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x00cea900u, 0xc1600000u, 0x0086c0efu, 0x91b00204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Ppsa02527R8UnormStorageTextureDescriptor() { - return {{0x00c7d500u, 0xc0100000u, 0x0086c0efu, 0x91b00204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x00c7d500u, 0xc0100000u, 0x0086c0efu, 0x91b00204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource BasicYzwxStorageTextureDescriptor() { - return {{0x00627801u, 0xc4d00000u, 0x0001c001u, 0x900009f5u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x00627801u, 0xc4d00000u, 0x0001c001u, 0x900009f5u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource BasicArrayStorageTextureResource() { - auto resource = BasicLinearStorageTextureResource(); - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - return resource; + auto resource = BasicLinearStorageTextureResource(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + return resource; } ShaderTextureResource BasicArrayStorageTextureDescriptor() { - return {{0x20179000u, 0x03800000u, 0x00000000u, 0xd1b00f2eu, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x20179000u, 0x03800000u, 0x00000000u, 0xd1b00f2eu, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource BasicUintArrayStorageTextureResource() { - auto resource = BasicArrayStorageTextureResource(); - resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - return resource; + auto resource = BasicArrayStorageTextureResource(); + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + return resource; } ShaderTextureResource BasicUintArrayStorageTextureDescriptor() { - return {{0x20179200u, 0x01400000u, 0x00000000u, 0xd1b00204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x20179200u, 0x01400000u, 0x00000000u, 0xd1b00204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Standard4KBUintArrayStorageTextureDescriptor() { - return {{0x006c6600u, 0x01400000u, 0x00000000u, 0xd0500204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x006c6600u, 0x01400000u, 0x00000000u, 0xd0500204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Standard64KBStorageTextureDescriptor() { - return {{0x011fab00u, 0xc3800000u, 0x0003c003u, 0xd0900facu, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x011fab00u, 0xc3800000u, 0x0003c003u, 0xd0900facu, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource Ppsa14053DepthTileStorageTextureResource() { - return BasicUintArrayStorageTextureResource(); + return BasicUintArrayStorageTextureResource(); } ShaderTextureResource Ppsa14053DepthTileStorageTextureDescriptor() { - return {{0x20144c00u, 0x00500000u, 0x00000000u, 0xd1800204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x20144c00u, 0x00500000u, 0x00000000u, 0xd1800204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderTextureResource Ppsa10112D16StorageTextureDescriptor() { - return {{0x205b9000u, 0xc0700000u, 0x0021803bu, 0xd1800204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x205b9000u, 0xc0700000u, 0x0021803bu, 0xd1800204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource BasicUintVolumeStorageTextureResource() { - auto resource = BasicStorageTextureResource(); - resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - resource.read = false; - return resource; + auto resource = BasicStorageTextureResource(); + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + resource.read = false; + return resource; } ShaderTextureResource BasicUintVolumeStorageTextureDescriptor() { - return {{0x20180600u, 0xc0b00000u, 0x0003c003u, 0xa0000004u, 0x0000000fu, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x20180600u, 0xc0b00000u, 0x0003c003u, 0xa0000004u, 0x0000000fu, + 0x00700000u, 0x00000000u, 0x00000000u}}; } ShaderRecompiler::IR::ImageResource AtomicStorageTextureResource() { - auto resource = BasicLinearStorageTextureResource(); - resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - resource.read = true; - resource.atomic = true; - return resource; + auto resource = BasicLinearStorageTextureResource(); + resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; + resource.read = true; + resource.atomic = true; + return resource; } ShaderTextureResource AtomicStorageTextureDescriptor() { - return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u, 0x00700000u, - 0x00000000u, 0x00000000u}}; + return {{0x304bb700u, 0xc1400000u, 0x0000001fu, 0x91b00204u, 0x00000000u, + 0x00700000u, 0x00000000u, 0x00000000u}}; } -[[noreturn]] void RunStorageTextureDescriptorDeathCase(const char* kind) { - auto resource = BasicStorageTextureResource(); - auto descriptor = BasicStorageTextureDescriptor(); - if (std::strcmp(kind, "linear-rgb1-read") == 0) { - resource = BasicLinearStorageTextureResource(); - descriptor = BasicLinearStorageTextureDescriptor(); - resource.read = true; - } else if (std::strcmp(kind, "bgra-read") == 0) { - resource = BasicBgraStorageTextureResource(); - descriptor = BasicBgraStorageTextureDescriptor(); - resource.read = true; - } else if (std::strcmp(kind, "r16-float-read") == 0) { - resource = BasicBgraStorageTextureResource(); - descriptor = Ppsa02527R16FloatStorageTextureDescriptor(); - resource.read = true; - } else if (std::strcmp(kind, "r8-unorm-read") == 0) { - resource = BasicBgraStorageTextureResource(); - descriptor = Ppsa02527R8UnormStorageTextureDescriptor(); - resource.read = true; - } else if (std::strcmp(kind, "yzwx-read") == 0) { - resource = BasicLinearStorageTextureResource(); - descriptor = BasicYzwxStorageTextureDescriptor(); - resource.read = true; - } else if (std::strcmp(kind, "reserved-swizzle") == 0) { - resource = BasicLinearStorageTextureResource(); - descriptor = BasicLinearStorageTextureDescriptor(); - descriptor.fields[3] = (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 2); - } else if (std::strcmp(kind, "resource") == 0) { - resource.written = false; - } else if (std::strcmp(kind, "type") == 0) { - descriptor.fields[3] = (descriptor.fields[3] & 0x0fffffffu) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - } else if (std::strcmp(kind, "standard256b-volume") == 0) { - descriptor.fields[3] = (descriptor.fields[3] & ~(0x1fu << 20u)) | - (static_cast(Prospero::TileMode::kStandard256B) << 20u); - } else if (std::strcmp(kind, "mip") == 0) { - descriptor.fields[3] |= 1u << 16u; - } else if (std::strcmp(kind, "swizzle") == 0) { - descriptor.fields[3] = (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 1); - } else if (std::strcmp(kind, "array-base-out-of-range") == 0) { - resource = BasicArrayStorageTextureResource(); - descriptor = BasicArrayStorageTextureDescriptor(); - descriptor.fields[4] |= 1u << 16u; - } else if (std::strcmp(kind, "reserved") == 0) { - descriptor.fields[1] |= 1u << 29u; - } else if (std::strcmp(kind, "uint-format") == 0) { - descriptor.fields[1] = - (descriptor.fields[1] & ~0x1ff00000u) | - (static_cast(Prospero::BufferFormat::k8_8_8_8UInt) << 20u); - } else if (std::strcmp(kind, "uint-resource-float-format") == 0) { - resource = BasicUintArrayStorageTextureResource(); - descriptor = BasicArrayStorageTextureDescriptor(); - } else if (std::strcmp(kind, "atomic-format") == 0) { - resource = AtomicStorageTextureResource(); - descriptor = AtomicStorageTextureDescriptor(); - descriptor.fields[1] = (descriptor.fields[1] & ~0x1ff00000u) | - (static_cast(Prospero::BufferFormat::k8UInt) << 20u); - } else if (std::strcmp(kind, "depth-tile-read") == 0) { - resource = Ppsa14053DepthTileStorageTextureResource(); - descriptor = Ppsa14053DepthTileStorageTextureDescriptor(); - resource.read = true; - } else if (std::strcmp(kind, "depth-tile-extent") == 0) { - resource = Ppsa14053DepthTileStorageTextureResource(); - descriptor = Ppsa14053DepthTileStorageTextureDescriptor(); - resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - descriptor.fields[3] = (descriptor.fields[3] & 0x0fffffffu) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - descriptor.fields[4] |= 1u; - } else if (std::strcmp(kind, "depth-tile-fmask") == 0) { - resource = BasicArrayStorageTextureResource(); - descriptor = Ppsa10112D16StorageTextureDescriptor(); - descriptor.fields[1] = - (descriptor.fields[1] & ~(0x1ffu << 20u)) | - (static_cast(Prospero::BufferFormat::kFmask8_S4_F4) << 20u); - } else { - std::_Exit(0x7e); - } - ValidateStorageTexture(resource, descriptor, 0x10000); - std::_Exit(0x7f); +[[noreturn]] void RunStorageTextureDescriptorDeathCase(const char *kind) { + auto resource = BasicStorageTextureResource(); + auto descriptor = BasicStorageTextureDescriptor(); + if (std::strcmp(kind, "linear-rgb1-read") == 0) { + resource = BasicLinearStorageTextureResource(); + descriptor = BasicLinearStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "bgra-read") == 0) { + resource = BasicBgraStorageTextureResource(); + descriptor = BasicBgraStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "r16-float-read") == 0) { + resource = BasicBgraStorageTextureResource(); + descriptor = Ppsa02527R16FloatStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "r8-unorm-read") == 0) { + resource = BasicBgraStorageTextureResource(); + descriptor = Ppsa02527R8UnormStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "yzwx-read") == 0) { + resource = BasicLinearStorageTextureResource(); + descriptor = BasicYzwxStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "reserved-swizzle") == 0) { + resource = BasicLinearStorageTextureResource(); + descriptor = BasicLinearStorageTextureDescriptor(); + descriptor.fields[3] = + (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 2); + } else if (std::strcmp(kind, "resource") == 0) { + resource.written = false; + } else if (std::strcmp(kind, "type") == 0) { + descriptor.fields[3] = + (descriptor.fields[3] & 0x0fffffffu) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + } else if (std::strcmp(kind, "standard256b-volume") == 0) { + descriptor.fields[3] = + (descriptor.fields[3] & ~(0x1fu << 20u)) | + (static_cast(Prospero::TileMode::kStandard256B) << 20u); + } else if (std::strcmp(kind, "mip") == 0) { + descriptor.fields[3] |= 1u << 16u; + } else if (std::strcmp(kind, "swizzle") == 0) { + descriptor.fields[3] = + (descriptor.fields[3] & ~0xfffu) | DstSel(4, 5, 6, 1); + } else if (std::strcmp(kind, "array-base-out-of-range") == 0) { + resource = BasicArrayStorageTextureResource(); + descriptor = BasicArrayStorageTextureDescriptor(); + descriptor.fields[4] |= 1u << 16u; + } else if (std::strcmp(kind, "reserved") == 0) { + descriptor.fields[1] |= 1u << 29u; + } else if (std::strcmp(kind, "uint-format") == 0) { + descriptor.fields[1] = + (descriptor.fields[1] & ~0x1ff00000u) | + (static_cast(Prospero::BufferFormat::k8_8_8_8UInt) << 20u); + } else if (std::strcmp(kind, "uint-resource-float-format") == 0) { + resource = BasicUintArrayStorageTextureResource(); + descriptor = BasicArrayStorageTextureDescriptor(); + } else if (std::strcmp(kind, "atomic-format") == 0) { + resource = AtomicStorageTextureResource(); + descriptor = AtomicStorageTextureDescriptor(); + descriptor.fields[1] = + (descriptor.fields[1] & ~0x1ff00000u) | + (static_cast(Prospero::BufferFormat::k8UInt) << 20u); + } else if (std::strcmp(kind, "depth-tile-read") == 0) { + resource = Ppsa14053DepthTileStorageTextureResource(); + descriptor = Ppsa14053DepthTileStorageTextureDescriptor(); + resource.read = true; + } else if (std::strcmp(kind, "depth-tile-extent") == 0) { + resource = Ppsa14053DepthTileStorageTextureResource(); + descriptor = Ppsa14053DepthTileStorageTextureDescriptor(); + resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + descriptor.fields[3] = + (descriptor.fields[3] & 0x0fffffffu) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + descriptor.fields[4] |= 1u; + } else if (std::strcmp(kind, "depth-tile-fmask") == 0) { + resource = BasicArrayStorageTextureResource(); + descriptor = Ppsa10112D16StorageTextureDescriptor(); + descriptor.fields[1] = + (descriptor.fields[1] & ~(0x1ffu << 20u)) | + (static_cast(Prospero::BufferFormat::kFmask8_S4_F4) << 20u); + } else { + std::_Exit(0x7e); + } + ValidateStorageTexture(resource, descriptor, 0x10000); + std::_Exit(0x7f); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS void CheckBasicStorageTextureDescriptor() { - const auto descriptor = BasicStorageTextureDescriptor(); - Require("BasicStorageTexture", "descriptor", - descriptor.Base40() == 0x785d0000ull && descriptor.Width5() + 1u == 33 && - descriptor.Height5() + 1u == 33 && descriptor.Depth() + 1u == 33, - "basic 3D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000); + const auto descriptor = BasicStorageTextureDescriptor(); + Require("BasicStorageTexture", "descriptor", + descriptor.Base40() == 0x785d0000ull && + descriptor.Width5() + 1u == 33 && + descriptor.Height5() + 1u == 33 && descriptor.Depth() + 1u == 33, + "basic 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicStorageTextureResource(), descriptor, 0x10000); - const ShaderTextureResource extended {{0x204aca00u, 0xc4700000u, 0x000fc00fu, 0xa1b00facu, - 0x0000003fu, 0x00700000u, 0x006b0000u, 0x00204b0au}}; - auto extended_resource = BasicStorageTextureResource(); - extended_resource.read = false; - Require("BasicStorageTexture", "extended descriptor", - extended.fields[6] != 0 && extended.fields[7] != 0, - "extended 3D storage descriptor fixture is malformed"); - ValidateStorageTexture(extended_resource, extended, 0x400000); + const ShaderTextureResource extended{{0x204aca00u, 0xc4700000u, 0x000fc00fu, + 0xa1b00facu, 0x0000003fu, 0x00700000u, + 0x006b0000u, 0x00204b0au}}; + auto extended_resource = BasicStorageTextureResource(); + extended_resource.read = false; + Require("BasicStorageTexture", "extended descriptor", + extended.fields[6] != 0 && extended.fields[7] != 0, + "extended 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(extended_resource, extended, 0x400000); - const auto linear = BasicLinearStorageTextureDescriptor(); - Require("BasicStorageTexture", "linear descriptor", - linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 && - linear.Height5() + 1u == 2160 && linear.Depth() + 1u == 1 && - linear.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && - linear.TileMode() == Prospero::TileMode::kLinear, - "PPSA07429 linear 2D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicLinearStorageTextureResource(), linear, 0x1fa4000); + const auto linear = BasicLinearStorageTextureDescriptor(); + Require("BasicStorageTexture", "linear descriptor", + linear.Base40() == 0x4bcc40100ull && linear.Width5() + 1u == 3840 && + linear.Height5() + 1u == 2160 && linear.Depth() + 1u == 1 && + linear.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && + linear.TileMode() == Prospero::TileMode::kLinear, + "PPSA07429 linear 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicLinearStorageTextureResource(), linear, + 0x1fa4000); - const auto bgra = BasicBgraStorageTextureDescriptor(); - Require("BasicStorageTexture", "BGRA descriptor", - bgra.Base40() == 0x7c650000ull && bgra.Width5() + 1u == 1920 && - bgra.Height5() + 1u == 1080 && bgra.Depth() + 1u == 1 && - bgra.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && - bgra.TileMode() == Prospero::TileMode::kRenderTarget && - bgra.DstSelXYZW() == DstSel(6, 5, 4, 7), - "PPSA02604 BGRA 2D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000); + const auto bgra = BasicBgraStorageTextureDescriptor(); + Require("BasicStorageTexture", "BGRA descriptor", + bgra.Base40() == 0x7c650000ull && bgra.Width5() + 1u == 1920 && + bgra.Height5() + 1u == 1080 && bgra.Depth() + 1u == 1 && + bgra.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && + bgra.TileMode() == Prospero::TileMode::kRenderTarget && + bgra.DstSelXYZW() == DstSel(6, 5, 4, 7), + "PPSA02604 BGRA 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), bgra, 0x870000); - const auto r11g11b10 = Ppsa06228R11G11B10StorageTextureDescriptor(); - Require("BasicStorageTexture", "PPSA06228 R11G11B10 descriptor", - r11g11b10.Base40() == 0x10c6b50000ull && r11g11b10.Width5() + 1u == 1920 && - r11g11b10.Height5() + 1u == 1080 && r11g11b10.Depth() + 1u == 1 && - r11g11b10.Format() == - Prospero::BufferFormat::k11_11_10Float && - r11g11b10.TileMode() == Prospero::TileMode::kRenderTarget && - r11g11b10.DstSelXYZW() == DstSel(4, 5, 6, 1), - "PPSA06228 R11G11B10 storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10, 0x870000); - ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm, vk::Format::eB10G11R11UfloatPack32, - r11g11b10.DstSelXYZW()); + const auto r11g11b10 = Ppsa06228R11G11B10StorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA06228 R11G11B10 descriptor", + r11g11b10.Base40() == 0x10c6b50000ull && + r11g11b10.Width5() + 1u == 1920 && + r11g11b10.Height5() + 1u == 1080 && r11g11b10.Depth() + 1u == 1 && + r11g11b10.Format() == Prospero::BufferFormat::k11_11_10Float && + r11g11b10.TileMode() == Prospero::TileMode::kRenderTarget && + r11g11b10.DstSelXYZW() == DstSel(4, 5, 6, 1), + "PPSA06228 R11G11B10 storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), r11g11b10, + 0x870000); + ValidateStorageColorView(vk::Format::eB8G8R8A8Unorm, + vk::Format::eB10G11R11UfloatPack32, + r11g11b10.DstSelXYZW()); - const auto max_mip = Ppsa01530MaxMipStorageTextureDescriptor(); - Require("BasicStorageTexture", "PPSA01530 max-mip descriptor", - max_mip.Base40() == 0x4a4290000ull && max_mip.Width5() + 1u == 64 && - max_mip.Height5() + 1u == 64 && max_mip.Depth() + 1u == 1 && - max_mip.BaseLevel() == 0 && max_mip.LastLevel() == 0 && max_mip.MaxMip() == 5 && - max_mip.Format() == Prospero::BufferFormat::k32_32UInt && - max_mip.TileMode() == Prospero::TileMode::kRenderTarget && - max_mip.DstSelXYZW() == DstSel(4, 5, 0, 1), - "PPSA01530 max-mip storage descriptor fixture is malformed"); - ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), max_mip, 0x20000); - auto mip_one = max_mip; - mip_one.fields[3] |= (1u << 12u) | (1u << 16u); - Require("BasicStorageTexture", "PPSA01530 mip-one descriptor", - mip_one.BaseLevel() == 1 && mip_one.LastLevel() == 1 && mip_one.MaxMip() == 5, - "PPSA01530 mip-one storage descriptor fixture is malformed"); - ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), mip_one, 0x20000); + const auto max_mip = Ppsa01530MaxMipStorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA01530 max-mip descriptor", + max_mip.Base40() == 0x4a4290000ull && max_mip.Width5() + 1u == 64 && + max_mip.Height5() + 1u == 64 && max_mip.Depth() + 1u == 1 && + max_mip.BaseLevel() == 0 && max_mip.LastLevel() == 0 && + max_mip.MaxMip() == 5 && + max_mip.Format() == Prospero::BufferFormat::k32_32UInt && + max_mip.TileMode() == Prospero::TileMode::kRenderTarget && + max_mip.DstSelXYZW() == DstSel(4, 5, 0, 1), + "PPSA01530 max-mip storage descriptor fixture is malformed"); + ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), max_mip, + 0x20000); + auto mip_one = max_mip; + mip_one.fields[3] |= (1u << 12u) | (1u << 16u); + Require("BasicStorageTexture", "PPSA01530 mip-one descriptor", + mip_one.BaseLevel() == 1 && mip_one.LastLevel() == 1 && + mip_one.MaxMip() == 5, + "PPSA01530 mip-one storage descriptor fixture is malformed"); + ValidateStorageTexture(Ppsa01530MaxMipStorageTextureResource(), mip_one, + 0x20000); - const auto r16_float = Ppsa02527R16FloatStorageTextureDescriptor(); - Require("BasicStorageTexture", "PPSA02527 R16F descriptor", - r16_float.Base40() == 0xce350000ull && r16_float.Width5() + 1u == 1920 && - r16_float.Height5() + 1u == 1080 && r16_float.Depth() + 1u == 1 && - r16_float.Format() == Prospero::BufferFormat::k16Float && - r16_float.TileMode() == Prospero::TileMode::kRenderTarget && - r16_float.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA02527 R16F 2D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicBgraStorageTextureResource(), r16_float, 0x480000); + const auto r16_float = Ppsa02527R16FloatStorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA02527 R16F descriptor", + r16_float.Base40() == 0xce350000ull && + r16_float.Width5() + 1u == 1920 && + r16_float.Height5() + 1u == 1080 && r16_float.Depth() + 1u == 1 && + r16_float.Format() == Prospero::BufferFormat::k16Float && + r16_float.TileMode() == Prospero::TileMode::kRenderTarget && + r16_float.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA02527 R16F 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), r16_float, + 0x480000); - const auto r32_float = Ppsa02527R32FloatStorageTextureDescriptor(); - Require("BasicStorageTexture", "PPSA02527 R32F descriptor", - r32_float.Base40() == 0xcea90000ull && r32_float.Width5() + 1u == 960 && - r32_float.Height5() + 1u == 540 && r32_float.Depth() + 1u == 1 && - r32_float.Format() == Prospero::BufferFormat::k32Float && - r32_float.TileMode() == Prospero::TileMode::kRenderTarget && - r32_float.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA02527 R32F 2D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicBgraStorageTextureResource(), r32_float, 0x280000); + const auto r32_float = Ppsa02527R32FloatStorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA02527 R32F descriptor", + r32_float.Base40() == 0xcea90000ull && + r32_float.Width5() + 1u == 960 && + r32_float.Height5() + 1u == 540 && r32_float.Depth() + 1u == 1 && + r32_float.Format() == Prospero::BufferFormat::k32Float && + r32_float.TileMode() == Prospero::TileMode::kRenderTarget && + r32_float.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA02527 R32F 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), r32_float, + 0x280000); - const auto r8_unorm = Ppsa02527R8UnormStorageTextureDescriptor(); - Require("BasicStorageTexture", "PPSA02527 R8 UNORM descriptor", - r8_unorm.Base40() == 0xc7d50000ull && r8_unorm.Width5() + 1u == 960 && - r8_unorm.Height5() + 1u == 540 && r8_unorm.Depth() + 1u == 1 && - r8_unorm.Format() == Prospero::BufferFormat::k8UNorm && - r8_unorm.TileMode() == Prospero::TileMode::kRenderTarget && - r8_unorm.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA02527 R8 UNORM 2D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicBgraStorageTextureResource(), r8_unorm, 0xc0000); + const auto r8_unorm = Ppsa02527R8UnormStorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA02527 R8 UNORM descriptor", + r8_unorm.Base40() == 0xc7d50000ull && r8_unorm.Width5() + 1u == 960 && + r8_unorm.Height5() + 1u == 540 && r8_unorm.Depth() + 1u == 1 && + r8_unorm.Format() == Prospero::BufferFormat::k8UNorm && + r8_unorm.TileMode() == Prospero::TileMode::kRenderTarget && + r8_unorm.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA02527 R8 UNORM 2D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), r8_unorm, 0xc0000); - const auto yzwx = BasicYzwxStorageTextureDescriptor(); - Require("BasicStorageTexture", "YZWX descriptor", - yzwx.Base40() == 0x62780100ull && yzwx.Width5() + 1u == 8 && yzwx.Height5() + 1u == 8 && - yzwx.Depth() + 1u == 1 && - yzwx.Format() == - Prospero::BufferFormat::k32_32_32_32Float && - yzwx.TileMode() == Prospero::TileMode::kLinear && - yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4), - "PPSA04181 linear YZWX storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800); - auto all_swizzles = yzwx; - all_swizzles.fields[1] = - (all_swizzles.fields[1] & ~0x1ff00000u) | - (static_cast(Prospero::BufferFormat::k16_16_16_16Float) << 20u); - uint32_t valid_storage_swizzles = 0; - for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) { - if (!IsValidImageSwizzle(swizzle)) { - continue; - } - all_swizzles.fields[3] = (all_swizzles.fields[3] & ~0xfffu) | swizzle; - ValidateStorageTexture(BasicLinearStorageTextureResource(), all_swizzles, 0x800); - ValidateStorageColorView(vk::Format::eR16G16B16A16Sfloat, vk::Format::eR16G16B16A16Sfloat, - swizzle); - ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb, vk::Format::eR8G8B8A8Unorm, swizzle); - ValidateStorageColorView(vk::Format::eB8G8R8A8Srgb, vk::Format::eR8G8B8A8Unorm, swizzle); - valid_storage_swizzles++; - } - Require("BasicStorageTexture", "all write swizzles", valid_storage_swizzles == 1296, - "valid write-only storage image mappings were rejected"); + const auto yzwx = BasicYzwxStorageTextureDescriptor(); + Require("BasicStorageTexture", "YZWX descriptor", + yzwx.Base40() == 0x62780100ull && yzwx.Width5() + 1u == 8 && + yzwx.Height5() + 1u == 8 && yzwx.Depth() + 1u == 1 && + yzwx.Format() == Prospero::BufferFormat::k32_32_32_32Float && + yzwx.TileMode() == Prospero::TileMode::kLinear && + yzwx.DstSelXYZW() == DstSel(5, 6, 7, 4), + "PPSA04181 linear YZWX storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicLinearStorageTextureResource(), yzwx, 0x800); + auto all_swizzles = yzwx; + all_swizzles.fields[1] = + (all_swizzles.fields[1] & ~0x1ff00000u) | + (static_cast(Prospero::BufferFormat::k16_16_16_16Float) << 20u); + uint32_t valid_storage_swizzles = 0; + for (uint32_t swizzle = 0; swizzle <= 0xfffu; swizzle++) { + if (!IsValidImageSwizzle(swizzle)) { + continue; + } + all_swizzles.fields[3] = (all_swizzles.fields[3] & ~0xfffu) | swizzle; + ValidateStorageTexture(BasicLinearStorageTextureResource(), all_swizzles, + 0x800); + ValidateStorageColorView(vk::Format::eR16G16B16A16Sfloat, + vk::Format::eR16G16B16A16Sfloat, swizzle); + ValidateStorageColorView(vk::Format::eR8G8B8A8Srgb, + vk::Format::eR8G8B8A8Unorm, swizzle); + ValidateStorageColorView(vk::Format::eB8G8R8A8Srgb, + vk::Format::eR8G8B8A8Unorm, swizzle); + valid_storage_swizzles++; + } + Require("BasicStorageTexture", "all write swizzles", + valid_storage_swizzles == 1296, + "valid write-only storage image mappings were rejected"); - const ShaderTextureResource cube {{0x025bca00u, 0xc4700000u, 0x003fc03fu, 0xb1b00facu, - 0x00000005u, 0x00700080u, 0x00000000u, 0x00000000u}}; - auto cube_resource = BasicArrayStorageTextureResource(); - cube_resource.cube = true; - Require("BasicStorageTexture", "PPSA07429 cube descriptor", - cube.Base40() == 0x25bca0000ull && cube.Width5() + 1u == 256 && - cube.Height5() + 1u == 256 && cube.Depth() + 1u == 6 && - cube.BaseArray5() == 0 && cube.MaxMip() == 8 && - cube.Type() == Prospero::ImageType::kCube && - cube.Format() == Prospero::BufferFormat::k16_16_16_16Float && - cube.TileMode() == Prospero::TileMode::kRenderTarget && - cube.DstSelXYZW() == DstSel(4, 5, 6, 7), - "PPSA07429 cube storage descriptor fixture is malformed"); - ValidateStorageTexture(cube_resource, cube, 0x420000); + const ShaderTextureResource cube{{0x025bca00u, 0xc4700000u, 0x003fc03fu, + 0xb1b00facu, 0x00000005u, 0x00700080u, + 0x00000000u, 0x00000000u}}; + auto cube_resource = BasicArrayStorageTextureResource(); + cube_resource.cube = true; + Require("BasicStorageTexture", "PPSA07429 cube descriptor", + cube.Base40() == 0x25bca0000ull && cube.Width5() + 1u == 256 && + cube.Height5() + 1u == 256 && cube.Depth() + 1u == 6 && + cube.BaseArray5() == 0 && cube.MaxMip() == 8 && + cube.Type() == Prospero::ImageType::kCube && + cube.Format() == Prospero::BufferFormat::k16_16_16_16Float && + cube.TileMode() == Prospero::TileMode::kRenderTarget && + cube.DstSelXYZW() == DstSel(4, 5, 6, 7), + "PPSA07429 cube storage descriptor fixture is malformed"); + ValidateStorageTexture(cube_resource, cube, 0x420000); - const auto array = BasicArrayStorageTextureDescriptor(); - Require("BasicStorageTexture", "2D-array descriptor", - array.Base40() == 0x2017900000ull && array.Width5() + 1u == 1 && - array.Height5() + 1u == 1 && array.Depth() + 1u == 1 && array.BaseArray5() == 0 && - array.Type() == Prospero::ImageType::kColor2DArray && - array.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && - array.TileMode() == Prospero::TileMode::kRenderTarget && - array.DstSelXYZW() == DstSel(6, 5, 4, 7), - "PPSA21268 2D-array storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicArrayStorageTextureResource(), array, 0x10000); - const ShaderTextureResource mip_array {{0x20268d00u, 0xc4700000u, 0x001fc01fu, - 0xd1b11facu, 0x00000000u, 0x00700070u, - 0x00000000u, 0x00000000u}}; - Require("BasicStorageTexture", "PPSA14457 mip-one 2D-array descriptor", - mip_array.BaseLevel() == 1 && mip_array.LastLevel() == 1 && - mip_array.MaxMip() == 7 && - mip_array.Type() == Prospero::ImageType::kColor2DArray && - mip_array.Depth() == 0 && mip_array.BaseArray5() == 0, - "PPSA14457 mip-one 2D-array storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicArrayStorageTextureResource(), mip_array, 0x30000); + const auto array = BasicArrayStorageTextureDescriptor(); + Require("BasicStorageTexture", "2D-array descriptor", + array.Base40() == 0x2017900000ull && array.Width5() + 1u == 1 && + array.Height5() + 1u == 1 && array.Depth() + 1u == 1 && + array.BaseArray5() == 0 && + array.Type() == Prospero::ImageType::kColor2DArray && + array.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && + array.TileMode() == Prospero::TileMode::kRenderTarget && + array.DstSelXYZW() == DstSel(6, 5, 4, 7), + "PPSA21268 2D-array storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicArrayStorageTextureResource(), array, 0x10000); + const ShaderTextureResource mip_array{{0x20268d00u, 0xc4700000u, 0x001fc01fu, + 0xd1b11facu, 0x00000000u, 0x00700070u, + 0x00000000u, 0x00000000u}}; + Require("BasicStorageTexture", "PPSA14457 mip-one 2D-array descriptor", + mip_array.BaseLevel() == 1 && mip_array.LastLevel() == 1 && + mip_array.MaxMip() == 7 && + mip_array.Type() == Prospero::ImageType::kColor2DArray && + mip_array.Depth() == 0 && mip_array.BaseArray5() == 0, + "PPSA14457 mip-one 2D-array storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicArrayStorageTextureResource(), mip_array, + 0x30000); - const auto uint_array = BasicUintArrayStorageTextureDescriptor(); - Require("BasicStorageTexture", "uint 2D-array descriptor", - uint_array.Base40() == 0x2017920000ull && uint_array.Width5() + 1u == 1 && - uint_array.Height5() + 1u == 1 && uint_array.Depth() + 1u == 1 && - uint_array.BaseArray5() == 0 && - uint_array.Type() == Prospero::ImageType::kColor2DArray && - uint_array.Format() == Prospero::BufferFormat::k32UInt && - uint_array.TileMode() == - Prospero::TileMode::kRenderTarget && - uint_array.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA21268 uint 2D-array storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicUintArrayStorageTextureResource(), uint_array, 0x10000); + const auto uint_array = BasicUintArrayStorageTextureDescriptor(); + Require("BasicStorageTexture", "uint 2D-array descriptor", + uint_array.Base40() == 0x2017920000ull && + uint_array.Width5() + 1u == 1 && uint_array.Height5() + 1u == 1 && + uint_array.Depth() + 1u == 1 && uint_array.BaseArray5() == 0 && + uint_array.Type() == Prospero::ImageType::kColor2DArray && + uint_array.Format() == Prospero::BufferFormat::k32UInt && + uint_array.TileMode() == Prospero::TileMode::kRenderTarget && + uint_array.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA21268 uint 2D-array storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicUintArrayStorageTextureResource(), uint_array, + 0x10000); - const ShaderTextureResource standard256b {{ - 0x204e4900u, - 0x43c00000u, - 0x00004000u, - 0x90100facu, - 0x00000000u, - 0x00700000u, - 0x00000000u, - 0x00000000u, - }}; - auto standard256b_resource = BasicBgraStorageTextureResource(); - standard256b_resource.kind = ShaderRecompiler::IR::ResourceKind::StorageImageUint; - TileSizeAlign standard256b_size {}; - TileGetTextureTotalSize(standard256b.Format(), standard256b.Width5() + 1u, - standard256b.Height5() + 1u, standard256b.Depth() + 1u, - 1, standard256b.TileMode(), false, - standard256b_size); - Require("BasicStorageTexture", "Standard256B uint 2D descriptor", - standard256b.Width5() + 1u == 2 && standard256b.Height5() + 1u == 2 && - standard256b.Depth() + 1u == 1 && - standard256b.Type() == Prospero::ImageType::kColor2D && - standard256b.Format() == - Prospero::BufferFormat::k8_8_8_8UInt && - standard256b.TileMode() == - Prospero::TileMode::kStandard256B && - standard256b.DstSelXYZW() == DstSel(4, 5, 6, 7) && - standard256b_size.size == 0x100 && standard256b_size.align == 0x100, - "captured PPSA08511 Standard256B storage descriptor is malformed"); - ValidateStorageTexture(standard256b_resource, standard256b, standard256b_size.size); + const ShaderTextureResource standard256b{{ + 0x204e4900u, + 0x43c00000u, + 0x00004000u, + 0x90100facu, + 0x00000000u, + 0x00700000u, + 0x00000000u, + 0x00000000u, + }}; + auto standard256b_resource = BasicBgraStorageTextureResource(); + standard256b_resource.kind = + ShaderRecompiler::IR::ResourceKind::StorageImageUint; + TileSizeAlign standard256b_size{}; + TileGetTextureTotalSize(standard256b.Format(), standard256b.Width5() + 1u, + standard256b.Height5() + 1u, + standard256b.Depth() + 1u, 1, standard256b.TileMode(), + false, standard256b_size); + Require("BasicStorageTexture", "Standard256B uint 2D descriptor", + standard256b.Width5() + 1u == 2 && standard256b.Height5() + 1u == 2 && + standard256b.Depth() + 1u == 1 && + standard256b.Type() == Prospero::ImageType::kColor2D && + standard256b.Format() == Prospero::BufferFormat::k8_8_8_8UInt && + standard256b.TileMode() == Prospero::TileMode::kStandard256B && + standard256b.DstSelXYZW() == DstSel(4, 5, 6, 7) && + standard256b_size.size == 0x100 && + standard256b_size.align == 0x100, + "captured PPSA08511 Standard256B storage descriptor is malformed"); + ValidateStorageTexture(standard256b_resource, standard256b, + standard256b_size.size); - const auto standard4kb_array = Standard4KBUintArrayStorageTextureDescriptor(); - TileSizeAlign standard4kb_size {}; - TileGetTextureTotalSize(standard4kb_array.Format(), 1, 1, 1, 1, - standard4kb_array.TileMode(), false, standard4kb_size); - Require( - "BasicStorageTexture", "Standard4KB uint 2D-array descriptor", - standard4kb_array.Type() == Prospero::ImageType::kColor2DArray && - standard4kb_array.Format() == Prospero::BufferFormat::k32UInt && - standard4kb_array.TileMode() == - Prospero::TileMode::kStandard4KB && - standard4kb_array.DstSelXYZW() == DstSel(4, 0, 0, 1) && - standard4kb_size.size == 0x1000 && standard4kb_size.align == 0x1000, - "captured Standard4KB uint 2D-array descriptor is malformed"); - ValidateStorageTexture(BasicUintArrayStorageTextureResource(), standard4kb_array, - standard4kb_size.size); + const auto standard4kb_array = Standard4KBUintArrayStorageTextureDescriptor(); + TileSizeAlign standard4kb_size{}; + TileGetTextureTotalSize(standard4kb_array.Format(), 1, 1, 1, 1, + standard4kb_array.TileMode(), false, + standard4kb_size); + Require( + "BasicStorageTexture", "Standard4KB uint 2D-array descriptor", + standard4kb_array.Type() == Prospero::ImageType::kColor2DArray && + standard4kb_array.Format() == Prospero::BufferFormat::k32UInt && + standard4kb_array.TileMode() == Prospero::TileMode::kStandard4KB && + standard4kb_array.DstSelXYZW() == DstSel(4, 0, 0, 1) && + standard4kb_size.size == 0x1000 && standard4kb_size.align == 0x1000, + "captured Standard4KB uint 2D-array descriptor is malformed"); + ValidateStorageTexture(BasicUintArrayStorageTextureResource(), + standard4kb_array, standard4kb_size.size); - auto based_standard4kb_array = standard4kb_array; - based_standard4kb_array.fields[0] = 0x006c6800u; - based_standard4kb_array.fields[4] = 0x00010001u; - TileSizeAlign based_standard4kb_size {}; - TileGetTextureTotalSize(based_standard4kb_array.Format(), 1, 1, - based_standard4kb_array.Depth() + 1u, 1, - based_standard4kb_array.TileMode(), false, based_standard4kb_size); - Require("BasicStorageTexture", "based Standard4KB array view", - based_standard4kb_array.Base40() == 0x6c680000ull && - based_standard4kb_array.BaseArray5() == 1 && based_standard4kb_array.Depth() == 1 && - based_standard4kb_size.size == 0x2000 && based_standard4kb_size.align == 0x1000, - "captured based Standard4KB array view is malformed"); - ValidateStorageTexture(BasicUintArrayStorageTextureResource(), based_standard4kb_array, - based_standard4kb_size.size); + auto based_standard4kb_array = standard4kb_array; + based_standard4kb_array.fields[0] = 0x006c6800u; + based_standard4kb_array.fields[4] = 0x00010001u; + TileSizeAlign based_standard4kb_size{}; + TileGetTextureTotalSize(based_standard4kb_array.Format(), 1, 1, + based_standard4kb_array.Depth() + 1u, 1, + based_standard4kb_array.TileMode(), false, + based_standard4kb_size); + Require("BasicStorageTexture", "based Standard4KB array view", + based_standard4kb_array.Base40() == 0x6c680000ull && + based_standard4kb_array.BaseArray5() == 1 && + based_standard4kb_array.Depth() == 1 && + based_standard4kb_size.size == 0x2000 && + based_standard4kb_size.align == 0x1000, + "captured based Standard4KB array view is malformed"); + ValidateStorageTexture(BasicUintArrayStorageTextureResource(), + based_standard4kb_array, based_standard4kb_size.size); - const auto standard64kb = Standard64KBStorageTextureDescriptor(); - TileSizeAlign standard64kb_size {}; - TileGetTextureTotalSize(standard64kb.Format(), standard64kb.Width5() + 1u, - standard64kb.Height5() + 1u, standard64kb.Depth() + 1u, - 1, standard64kb.TileMode(), false, - standard64kb_size); - Require("BasicStorageTexture", "Standard64KB 2D descriptor", - standard64kb.Type() == Prospero::ImageType::kColor2DArray && - standard64kb.Format() == - Prospero::BufferFormat::k8_8_8_8UNorm && - standard64kb.TileMode() == - Prospero::TileMode::kStandard64KB && - standard64kb.DstSelXYZW() == DstSel(4, 5, 6, 7) && - standard64kb_size.size == 0x10000 && standard64kb_size.align == 0x10000, - "captured Standard64KB storage descriptor is malformed"); - ValidateStorageTexture(BasicBgraStorageTextureResource(), standard64kb, standard64kb_size.size); + const auto standard64kb = Standard64KBStorageTextureDescriptor(); + TileSizeAlign standard64kb_size{}; + TileGetTextureTotalSize(standard64kb.Format(), standard64kb.Width5() + 1u, + standard64kb.Height5() + 1u, + standard64kb.Depth() + 1u, 1, standard64kb.TileMode(), + false, standard64kb_size); + Require("BasicStorageTexture", "Standard64KB 2D descriptor", + standard64kb.Type() == Prospero::ImageType::kColor2DArray && + standard64kb.Format() == Prospero::BufferFormat::k8_8_8_8UNorm && + standard64kb.TileMode() == Prospero::TileMode::kStandard64KB && + standard64kb.DstSelXYZW() == DstSel(4, 5, 6, 7) && + standard64kb_size.size == 0x10000 && + standard64kb_size.align == 0x10000, + "captured Standard64KB storage descriptor is malformed"); + ValidateStorageTexture(BasicBgraStorageTextureResource(), standard64kb, + standard64kb_size.size); - const auto uint_volume = BasicUintVolumeStorageTextureDescriptor(); - Require("BasicStorageTexture", "uint 3D descriptor", - uint_volume.Base40() == 0x2018060000ull && uint_volume.Width5() + 1u == 16 && - uint_volume.Height5() + 1u == 16 && uint_volume.Depth() + 1u == 16 && - uint_volume.BaseArray5() == 0 && - uint_volume.Type() == Prospero::ImageType::kColor3D && - uint_volume.Format() == Prospero::BufferFormat::k16UInt && - uint_volume.TileMode() == Prospero::TileMode::kLinear && - uint_volume.DstSelXYZW() == DstSel(4, 0, 0, 0), - "PPSA21268 uint 3D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume, 0x10000); + const auto uint_volume = BasicUintVolumeStorageTextureDescriptor(); + Require("BasicStorageTexture", "uint 3D descriptor", + uint_volume.Base40() == 0x2018060000ull && + uint_volume.Width5() + 1u == 16 && + uint_volume.Height5() + 1u == 16 && + uint_volume.Depth() + 1u == 16 && uint_volume.BaseArray5() == 0 && + uint_volume.Type() == Prospero::ImageType::kColor3D && + uint_volume.Format() == Prospero::BufferFormat::k16UInt && + uint_volume.TileMode() == Prospero::TileMode::kLinear && + uint_volume.DstSelXYZW() == DstSel(4, 0, 0, 0), + "PPSA21268 uint 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), uint_volume, + 0x10000); - const ShaderTextureResource tiled_uint_volume {{0x1ac0e530u, 0xc0b00000u, 0x0003c003u, - 0xa0500004u, 0x0000000fu, 0x00700000u, - 0x00000000u, 0x00000000u}}; - Require("BasicStorageTexture", "tiled uint 3D descriptor", - tiled_uint_volume.Width5() + 1u == 16 && tiled_uint_volume.Height5() + 1u == 16 && - tiled_uint_volume.Depth() + 1u == 16 && - tiled_uint_volume.Type() == Prospero::ImageType::kColor3D && - tiled_uint_volume.TileMode() == - Prospero::TileMode::kStandard4KB, - "captured tiled uint 3D storage descriptor fixture is malformed"); - ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), tiled_uint_volume, 0x2000); + const ShaderTextureResource tiled_uint_volume{ + {0x1ac0e530u, 0xc0b00000u, 0x0003c003u, 0xa0500004u, 0x0000000fu, + 0x00700000u, 0x00000000u, 0x00000000u}}; + Require("BasicStorageTexture", "tiled uint 3D descriptor", + tiled_uint_volume.Width5() + 1u == 16 && + tiled_uint_volume.Height5() + 1u == 16 && + tiled_uint_volume.Depth() + 1u == 16 && + tiled_uint_volume.Type() == Prospero::ImageType::kColor3D && + tiled_uint_volume.TileMode() == Prospero::TileMode::kStandard4KB, + "captured tiled uint 3D storage descriptor fixture is malformed"); + ValidateStorageTexture(BasicUintVolumeStorageTextureResource(), + tiled_uint_volume, 0x2000); - const auto depth_tile = Ppsa14053DepthTileStorageTextureDescriptor(); - Require("BasicStorageTexture", "PPSA14053 depth-tile descriptor", - depth_tile.Base40() == 0x20144c0000ull && depth_tile.Width5() + 1u == 1 && - depth_tile.Height5() + 1u == 1 && depth_tile.Depth() + 1u == 1 && - depth_tile.BaseArray5() == 0 && - depth_tile.Type() == Prospero::ImageType::kColor2DArray && - depth_tile.Format() == Prospero::BufferFormat::k8UInt && - depth_tile.TileMode() == Prospero::TileMode::kDepth && - depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA14053 write-only depth-tile storage descriptor fixture is " - "malformed"); - ValidateStorageTexture(Ppsa14053DepthTileStorageTextureResource(), depth_tile, 0x10000); - const auto d16_depth_tile = Ppsa10112D16StorageTextureDescriptor(); - Require( - "BasicStorageTexture", "PPSA10112 D16 depth-tile descriptor", - d16_depth_tile.Base40() == 0x205b900000ull && d16_depth_tile.Width5() + 1u == 240 && - d16_depth_tile.Height5() + 1u == 135 && d16_depth_tile.Depth() + 1u == 1 && - d16_depth_tile.Type() == Prospero::ImageType::kColor2DArray && - d16_depth_tile.Format() == Prospero::BufferFormat::k16UNorm && - d16_depth_tile.TileMode() == Prospero::TileMode::kDepth && - d16_depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA10112 writable D16 depth-plane descriptor fixture is malformed"); - const auto d16_pitch = TileGetTexturePitch( - d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u, d16_depth_tile.TileMode()); - TileSizeAlign d16_size {}; - TileGetTextureTotalSize(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u, - d16_depth_tile.Height5() + 1u, d16_depth_tile.Depth() + 1u, 1, - d16_depth_tile.TileMode(), false, d16_size); - Require("BasicStorageTexture", "PPSA10112 D16 depth-tile footprint", - d16_pitch == 256 && d16_size.size == 0x20000 && d16_size.align == 0x10000, - "PPSA10112 writable D16 depth-plane footprint is incorrect"); - ValidateStorageTexture(BasicArrayStorageTextureResource(), d16_depth_tile, d16_size.size); - auto depth_tile_r32 = depth_tile; - depth_tile_r32.fields[1] = (depth_tile_r32.fields[1] & ~(0x1ffu << 20u)) | - (static_cast(Prospero::BufferFormat::k32UInt) << 20u); - depth_tile_r32.fields[3] = (depth_tile_r32.fields[3] & ~(0xfu << 28u)) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - auto depth_tile_r32_resource = Ppsa14053DepthTileStorageTextureResource(); - depth_tile_r32_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - ValidateStorageTexture(depth_tile_r32_resource, depth_tile_r32, 0x10000); - Require("BasicStorageTexture", "R32_UINT replicated write mapping", - IsValidImageSwizzle(DstSel(4, 4, 4, 4)), - "single-channel replicated destination selection was rejected"); + const auto depth_tile = Ppsa14053DepthTileStorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA14053 depth-tile descriptor", + depth_tile.Base40() == 0x20144c0000ull && + depth_tile.Width5() + 1u == 1 && depth_tile.Height5() + 1u == 1 && + depth_tile.Depth() + 1u == 1 && depth_tile.BaseArray5() == 0 && + depth_tile.Type() == Prospero::ImageType::kColor2DArray && + depth_tile.Format() == Prospero::BufferFormat::k8UInt && + depth_tile.TileMode() == Prospero::TileMode::kDepth && + depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA14053 write-only depth-tile storage descriptor fixture is " + "malformed"); + ValidateStorageTexture(Ppsa14053DepthTileStorageTextureResource(), depth_tile, + 0x10000); + const auto d16_depth_tile = Ppsa10112D16StorageTextureDescriptor(); + Require("BasicStorageTexture", "PPSA10112 D16 depth-tile descriptor", + d16_depth_tile.Base40() == 0x205b900000ull && + d16_depth_tile.Width5() + 1u == 240 && + d16_depth_tile.Height5() + 1u == 135 && + d16_depth_tile.Depth() + 1u == 1 && + d16_depth_tile.Type() == Prospero::ImageType::kColor2DArray && + d16_depth_tile.Format() == Prospero::BufferFormat::k16UNorm && + d16_depth_tile.TileMode() == Prospero::TileMode::kDepth && + d16_depth_tile.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA10112 writable D16 depth-plane descriptor fixture is malformed"); + const auto d16_pitch = + TileGetTexturePitch(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u, + d16_depth_tile.TileMode()); + TileSizeAlign d16_size{}; + TileGetTextureTotalSize(d16_depth_tile.Format(), d16_depth_tile.Width5() + 1u, + d16_depth_tile.Height5() + 1u, + d16_depth_tile.Depth() + 1u, 1, + d16_depth_tile.TileMode(), false, d16_size); + Require("BasicStorageTexture", "PPSA10112 D16 depth-tile footprint", + d16_pitch == 256 && d16_size.size == 0x20000 && + d16_size.align == 0x10000, + "PPSA10112 writable D16 depth-plane footprint is incorrect"); + ValidateStorageTexture(BasicArrayStorageTextureResource(), d16_depth_tile, + d16_size.size); + auto depth_tile_r32 = depth_tile; + depth_tile_r32.fields[1] = + (depth_tile_r32.fields[1] & ~(0x1ffu << 20u)) | + (static_cast(Prospero::BufferFormat::k32UInt) << 20u); + depth_tile_r32.fields[3] = + (depth_tile_r32.fields[3] & ~(0xfu << 28u)) | + (static_cast(Prospero::ImageType::kColor2D) << 28u); + auto depth_tile_r32_resource = Ppsa14053DepthTileStorageTextureResource(); + depth_tile_r32_resource.dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2D; + ValidateStorageTexture(depth_tile_r32_resource, depth_tile_r32, 0x10000); + Require("BasicStorageTexture", "R32_UINT replicated write mapping", + IsValidImageSwizzle(DstSel(4, 4, 4, 4)), + "single-channel replicated destination selection was rejected"); - const auto atomic = AtomicStorageTextureDescriptor(); - Require("BasicStorageTexture", "atomic R32_UINT descriptor", - atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 && atomic.Depth() + 1u == 1 && - atomic.Type() == Prospero::ImageType::kColor2D && - atomic.Format() == Prospero::BufferFormat::k32UInt && - atomic.DstSelXYZW() == DstSel(4, 0, 0, 1), - "PPSA22102 image-atomic descriptor fixture is malformed"); - ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000); + const auto atomic = AtomicStorageTextureDescriptor(); + Require("BasicStorageTexture", "atomic R32_UINT descriptor", + atomic.Width5() + 1u == 128 && atomic.Height5() + 1u == 1 && + atomic.Depth() + 1u == 1 && + atomic.Type() == Prospero::ImageType::kColor2D && + atomic.Format() == Prospero::BufferFormat::k32UInt && + atomic.DstSelXYZW() == DstSel(4, 0, 0, 1), + "PPSA22102 image-atomic descriptor fixture is malformed"); + ValidateStorageTexture(AtomicStorageTextureResource(), atomic, 0x10000); - char path[MAX_PATH] {}; - Require("BasicStorageTexture", "host", GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, - "GetModuleFileName failed"); - for (const char* kind: {"resource", - "type", - "standard256b-volume", - "mip", - "swizzle", - "linear-rgb1-read", - "bgra-read", - "r16-float-read", - "r8-unorm-read", - "yzwx-read", - "reserved-swizzle", - "array-base-out-of-range", - "reserved", - "uint-format", - "uint-resource-float-format", - "atomic-format", - "depth-tile-read", - "depth-tile-extent", - "depth-tile-fmask"}) { - std::string command = - std::string("\"") + path + "\" --storage-texture-descriptor-death " + kind; - std::vector mutable_command(command.begin(), command.end()); - mutable_command.push_back('\0'); - STARTUPINFOA startup {sizeof(startup)}; - PROCESS_INFORMATION process {}; - Require("BasicStorageTexture", "host", - CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, - CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != 0, - "CreateProcess failed"); - Require("BasicStorageTexture", "host", - WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, - "descriptor death case timed out"); - DWORD exit_code = 0; - const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; - CloseHandle(process.hThread); - CloseHandle(process.hProcess); - Require("BasicStorageTexture", "host", exited && exit_code == 321, - std::string(kind) + " storage descriptor did not report a fatal error"); - } - std::printf("[host] %-32s ok\n", "BasicStorageTextureDescriptor"); + char path[MAX_PATH]{}; + Require("BasicStorageTexture", "host", + GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, + "GetModuleFileName failed"); + for (const char *kind : + {"resource", "type", "standard256b-volume", "mip", "swizzle", + "linear-rgb1-read", "bgra-read", "r16-float-read", "r8-unorm-read", + "yzwx-read", "reserved-swizzle", "array-base-out-of-range", "reserved", + "uint-format", "uint-resource-float-format", "atomic-format", + "depth-tile-read", "depth-tile-extent", "depth-tile-fmask"}) { + std::string command = std::string("\"") + path + + "\" --storage-texture-descriptor-death " + kind; + std::vector mutable_command(command.begin(), command.end()); + mutable_command.push_back('\0'); + STARTUPINFOA startup{sizeof(startup)}; + PROCESS_INFORMATION process{}; + Require("BasicStorageTexture", "host", + CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, + FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, + &process) != 0, + "CreateProcess failed"); + Require("BasicStorageTexture", "host", + WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, + "descriptor death case timed out"); + DWORD exit_code = 0; + const bool exited = GetExitCodeProcess(process.hProcess, &exit_code) != 0; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + Require("BasicStorageTexture", "host", exited && exit_code == 321, + std::string(kind) + + " storage descriptor did not report a fatal error"); + } + std::printf("[host] %-32s ok\n", "BasicStorageTextureDescriptor"); } #endif void CheckStorageTextureLinearUploadLayout() { - constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm; - constexpr uint32_t width = 3840; - constexpr uint32_t height = 2160; - constexpr uint32_t depth = 1; - constexpr auto tile = Prospero::TileMode::kLinear; - const auto pitch = TileGetTexturePitch(format, width, tile); - TileSizeAlign total {}; - TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size, true, - false, "StorageTextureLinearTest"); - const auto regions = TextureBuildImageCopies(layout); - Require("StorageTextureLinearUpload", "layout", - pitch == width && total.size == 0x1fa4000 && total.align == 256 && - layout.surface.description.tile_mode == tile && layout.pitch == width && - layout.slice_stride == total.size && - regions.size() == 1 && regions[0].bufferOffset == 0 && - regions[0].imageExtent.width == width && regions[0].imageExtent.height == height && - regions[0].bufferRowLength == width, - "linear RGBA8 storage upload lost Prospero pitch or allocation size"); - std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload"); + constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm; + constexpr uint32_t width = 3840; + constexpr uint32_t height = 2160; + constexpr uint32_t depth = 1; + constexpr auto tile = Prospero::TileMode::kLinear; + const auto pitch = TileGetTexturePitch(format, width, tile); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total); + const auto layout = + TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size, + true, false, "StorageTextureLinearTest"); + const auto regions = TextureBuildImageCopies(layout); + Require("StorageTextureLinearUpload", "layout", + pitch == width && total.size == 0x1fa4000 && total.align == 256 && + layout.surface.description.tile_mode == tile && + layout.pitch == width && layout.slice_stride == total.size && + regions.size() == 1 && regions[0].bufferOffset == 0 && + regions[0].imageExtent.width == width && + regions[0].imageExtent.height == height && + regions[0].bufferRowLength == width, + "linear RGBA8 storage upload lost Prospero pitch or allocation size"); + std::printf("[host] %-32s ok\n", "StorageTextureLinearUpload"); } void CheckStorageTextureDepthTileUploadLayout() { - constexpr auto format = Prospero::BufferFormat::k8UInt; - constexpr uint32_t width = 1; - constexpr uint32_t height = 1; - constexpr uint32_t depth = 1; - constexpr auto tile = Prospero::TileMode::kDepth; - const auto pitch = TileGetTexturePitch(format, width, tile); - TileSizeAlign slice {}; - TileSizeAlign total {}; - TileSizeOffset level {}; - TilePaddedSize padded {}; - TileGetTextureSize(format, width, height, 1, tile, &slice, &level, &padded); - TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size, true, - false, "StorageTextureDepthTileTest"); - const auto regions = TextureBuildImageCopies(layout); - Require("StorageTextureDepthTileUpload", "PPSA14053 layout", - pitch == 256 && padded.width == 256 && padded.height == 256 && slice.size == 0x10000 && - slice.align == 0x10000 && level.size == slice.size && level.offset == 0 && - total.size == slice.size && total.align == slice.align && - layout.surface.description.tile_mode == tile && - layout.surface.texture.block.family == TileBlockFamily::Depth64KB && - layout.pitch == pitch && - layout.slice_stride == pitch && layout.source_slice_stride == total.size && - layout.mips[0].size == pitch && - layout.surface.mips[0].size == total.size && regions.size() == 1 && - regions[0].bufferOffset == 0 && regions[0].imageExtent.width == width && - regions[0].imageExtent.height == height && regions[0].bufferRowLength == pitch, - "1x1 R8_UINT depth tile lost its 64 KiB source footprint"); - std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload"); + constexpr auto format = Prospero::BufferFormat::k8UInt; + constexpr uint32_t width = 1; + constexpr uint32_t height = 1; + constexpr uint32_t depth = 1; + constexpr auto tile = Prospero::TileMode::kDepth; + const auto pitch = TileGetTexturePitch(format, width, tile); + TileSizeAlign slice{}; + TileSizeAlign total{}; + TileSizeOffset level{}; + TilePaddedSize padded{}; + TileGetTextureSize(format, width, height, 1, tile, &slice, &level, &padded); + TileGetTextureTotalSize(format, width, height, depth, 1, tile, false, total); + const auto layout = + TextureCalcUploadLayout(format, width, height, 1, depth, tile, total.size, + true, false, "StorageTextureDepthTileTest"); + const auto regions = TextureBuildImageCopies(layout); + Require("StorageTextureDepthTileUpload", "PPSA14053 layout", + pitch == 256 && padded.width == 256 && padded.height == 256 && + slice.size == 0x10000 && slice.align == 0x10000 && + level.size == slice.size && level.offset == 0 && + total.size == slice.size && total.align == slice.align && + layout.surface.description.tile_mode == tile && + layout.surface.texture.block.family == + TileBlockFamily::Depth64KB && + layout.pitch == pitch && layout.slice_stride == pitch && + layout.source_slice_stride == total.size && + layout.mips[0].size == pitch && + layout.surface.mips[0].size == total.size && + regions.size() == 1 && regions[0].bufferOffset == 0 && + regions[0].imageExtent.width == width && + regions[0].imageExtent.height == height && + regions[0].bufferRowLength == pitch, + "1x1 R8_UINT depth tile lost its 64 KiB source footprint"); + std::printf("[host] %-32s ok\n", "StorageTextureDepthTileUpload"); } void CheckStorageImageSwizzleSpecializationId() { - std::array code {}; - HW::ComputeShaderInfo regs {}; - regs.cs_regs.data_addr = reinterpret_cast(code.data()); + std::array code{}; + HW::ComputeShaderInfo regs{}; + regs.cs_regs.data_addr = reinterpret_cast(code.data()); - ShaderRecompiler::IR::Program identity_program; - identity_program.binding_layout_complete = true; - ShaderRecompiler::IR::ImageResource image; - image.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; - image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; - identity_program.info.images.push_back(image); - auto rgb1_program = identity_program; - rgb1_program.info.images[0].storage_swizzle = DstSel(4, 5, 6, 1); + ShaderRecompiler::IR::Program identity_program; + identity_program.binding_layout_complete = true; + ShaderRecompiler::IR::ImageResource image; + image.kind = ShaderRecompiler::IR::ResourceKind::StorageImage; + image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + identity_program.info.images.push_back(image); + auto rgb1_program = identity_program; + rgb1_program.info.images[0].storage_swizzle = DstSel(4, 5, 6, 1); - const auto resources = std::make_shared(); - ShaderComputeInputInfo identity_info; - identity_info.stage.program = - std::make_shared(identity_program); - identity_info.stage.resources = resources; - auto rgb1_info = identity_info; - rgb1_info.stage.program = std::make_shared(rgb1_program); + const auto resources = + std::make_shared(); + ShaderComputeInputInfo identity_info; + identity_info.stage.program = + std::make_shared(identity_program); + identity_info.stage.resources = resources; + auto rgb1_info = identity_info; + rgb1_info.stage.program = + std::make_shared(rgb1_program); - const auto identity_id = ShaderGetIdCS(regs, identity_info, true); - const auto rgb1_id = ShaderGetIdCS(regs, rgb1_info, true); - Require("StorageImageSwizzleSpecializationId", "pipeline cache key", - identity_id != rgb1_id && identity_id.ids.size() == rgb1_id.ids.size(), - "storage swizzle-specialized SPIR-V variants share a pipeline ID"); - std::printf("[host] %-32s ok\n", "StorageImageSwizzlePipelineId"); + const auto identity_id = ShaderGetIdCS(regs, identity_info, true); + const auto rgb1_id = ShaderGetIdCS(regs, rgb1_info, true); + Require("StorageImageSwizzleSpecializationId", "pipeline cache key", + identity_id != rgb1_id && + identity_id.ids.size() == rgb1_id.ids.size(), + "storage swizzle-specialized SPIR-V variants share a pipeline ID"); + std::printf("[host] %-32s ok\n", "StorageImageSwizzlePipelineId"); } void CheckStorageTextureVolumeUploadLayout() { - constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float; - constexpr uint32_t width = 33; - constexpr uint32_t height = 33; - constexpr uint32_t depth = 33; - const auto pitch = TileGetTexturePitch( - format, width, Prospero::TileMode::kRenderTarget); - TileSizeAlign total {}; - TileGetTextureTotalSize(format, width, height, depth, 1, - Prospero::TileMode::kRenderTarget, true, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, 1, depth, - Prospero::TileMode::kRenderTarget, - total.size, true, true, "StorageTextureVolumeTest"); - const auto regions = TextureBuildImageCopies(layout); - Require("StorageTextureVolumeUpload", "layout", - pitch == 128 && total.size == 0x210000 && layout.slice_stride == 0x2208 && - layout.source_slice_stride == 0 && layout.mips[0].size == 0x2208 && - layout.surface.mips[0].size == 0x10000 && regions.size() == depth, - "3D render-target upload did not preserve its compact linear layout"); + constexpr auto format = Prospero::BufferFormat::k16_16_16_16Float; + constexpr uint32_t width = 33; + constexpr uint32_t height = 33; + constexpr uint32_t depth = 33; + const auto pitch = + TileGetTexturePitch(format, width, Prospero::TileMode::kRenderTarget); + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, 1, + Prospero::TileMode::kRenderTarget, true, total); + const auto layout = TextureCalcUploadLayout( + format, width, height, 1, depth, Prospero::TileMode::kRenderTarget, + total.size, true, true, "StorageTextureVolumeTest"); + const auto regions = TextureBuildImageCopies(layout); + Require( + "StorageTextureVolumeUpload", "layout", + pitch == 128 && total.size == 0x210000 && layout.slice_stride == 0x2208 && + layout.source_slice_stride == 0 && layout.mips[0].size == 0x2208 && + layout.surface.mips[0].size == 0x10000 && regions.size() == depth, + "3D render-target upload did not preserve its compact linear layout"); - std::vector infos; - Require("StorageTextureVolumeUpload", "GPU records", - TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos) && - infos.size() == depth, - "3D render-target GPU records were not built"); - for (const uint32_t z: {0u, 1u, depth - 1u}) { - Require("StorageTextureVolumeUpload", "slice offsets", - infos[z].linear_offset == static_cast(z) * 0x2208 && - infos[z].tiled_offset == static_cast(z) * 0x10000 && - infos[z].surface_z == z && infos[z].pitch == width, - "volume slice lost its linear stride, block slice, or Z swizzle"); - } - std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload"); + std::vector infos; + Require("StorageTextureVolumeUpload", "GPU records", + TextureBuildGpuTileInfos(total.size, regions, layout, 1, infos) && + infos.size() == depth, + "3D render-target GPU records were not built"); + for (const uint32_t z : {0u, 1u, depth - 1u}) { + Require("StorageTextureVolumeUpload", "slice offsets", + infos[z].linear_offset == static_cast(z) * 0x2208 && + infos[z].tiled_offset == static_cast(z) * 0x10000 && + infos[z].surface_z == z && infos[z].pitch == width, + "volume slice lost its linear stride, block slice, or Z swizzle"); + } + std::printf("[host] %-32s ok\n", "StorageTextureVolumeUpload"); } void CheckStorageTextureVolumeMipRegions() { - constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm; - constexpr uint32_t width = 8; - constexpr uint32_t height = 4; - constexpr uint32_t depth = 5; - constexpr uint32_t levels = 3; - constexpr auto tile = Prospero::TileMode::kLinear; - TileSizeAlign total {}; - TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, total); - const auto layout = - TextureCalcUploadLayout(format, width, height, levels, depth, tile, total.size, true, - true, "StorageTextureVolumeMipTest"); - const auto copies = TextureBuildImageCopies(layout); + constexpr auto format = Prospero::BufferFormat::k8_8_8_8UNorm; + constexpr uint32_t width = 8; + constexpr uint32_t height = 4; + constexpr uint32_t depth = 5; + constexpr uint32_t levels = 3; + constexpr auto tile = Prospero::TileMode::kLinear; + TileSizeAlign total{}; + TileGetTextureTotalSize(format, width, height, depth, levels, tile, true, + total); + const auto layout = TextureCalcUploadLayout( + format, width, height, levels, depth, tile, total.size, true, true, + "StorageTextureVolumeMipTest"); + const auto copies = TextureBuildImageCopies(layout); - bool valid = copies.size() == 8; - size_t index = 0; - for (uint32_t level = 0; level < levels && valid; level++) { - const uint32_t mip_depth = std::max(depth >> level, 1u); - const uint32_t mip_width = std::max(width >> level, 1u); - const uint32_t mip_height = std::max(height >> level, 1u); - for (uint32_t z = 0; z < mip_depth; z++, index++) { - const auto& copy = copies[index]; - valid &= - copy.imageSubresource.mipLevel == level && - copy.imageOffset.z == static_cast(z) && copy.imageExtent.width == mip_width && - copy.imageExtent.height == mip_height && - copy.bufferOffset == layout.mips[level].offset + z * layout.slice_stride && - copy.bufferRowLength == layout.mips[level].row_length && - copy.bufferImageHeight == layout.mips[level].image_height; - } - } - valid &= index == copies.size(); - Require("StorageTextureVolumeMipRegions", "per-mip depth", valid, - "direction-neutral image copies did not shrink depth or preserve " - "Vulkan Z coordinates"); - std::printf("[host] %-32s ok\n", "StorageTextureVolumeMipRegions"); + bool valid = copies.size() == 8; + size_t index = 0; + for (uint32_t level = 0; level < levels && valid; level++) { + const uint32_t mip_depth = std::max(depth >> level, 1u); + const uint32_t mip_width = std::max(width >> level, 1u); + const uint32_t mip_height = std::max(height >> level, 1u); + for (uint32_t z = 0; z < mip_depth; z++, index++) { + const auto © = copies[index]; + valid &= copy.imageSubresource.mipLevel == level && + copy.imageOffset.z == static_cast(z) && + copy.imageExtent.width == mip_width && + copy.imageExtent.height == mip_height && + copy.bufferOffset == + layout.mips[level].offset + z * layout.slice_stride && + copy.bufferRowLength == layout.mips[level].row_length && + copy.bufferImageHeight == layout.mips[level].image_height; + } + } + valid &= index == copies.size(); + Require("StorageTextureVolumeMipRegions", "per-mip depth", valid, + "direction-neutral image copies did not shrink depth or preserve " + "Vulkan Z coordinates"); + std::printf("[host] %-32s ok\n", "StorageTextureVolumeMipRegions"); } void CheckStandard64RenderTargetTileRoundTrip() { - constexpr auto format = Prospero::BufferFormat::k32Float; - constexpr auto tile = Prospero::TileMode::kStandard64KB; + constexpr auto format = Prospero::BufferFormat::k32Float; + constexpr auto tile = Prospero::TileMode::kStandard64KB; - constexpr uint32_t observed_width = 3840; - constexpr uint32_t observed_height = 2160; - const auto observed_pitch = TileGetTexturePitch(format, observed_width, tile); - TileSizeAlign observed {}; - TileGetTextureSize(format, observed_width, observed_height, 1, tile, &observed, - nullptr, nullptr); - Require("Standard64RenderTarget", "observed layout", - observed_pitch == 3840 && observed.size == 0x1fe0000 && observed.align == 0x10000, - "PPSA02721 Standard64KB render-target footprint changed"); + constexpr uint32_t observed_width = 3840; + constexpr uint32_t observed_height = 2160; + const auto observed_pitch = TileGetTexturePitch(format, observed_width, tile); + TileSizeAlign observed{}; + TileGetTextureSize(format, observed_width, observed_height, 1, tile, + &observed, nullptr, nullptr); + Require("Standard64RenderTarget", "observed layout", + observed_pitch == 3840 && observed.size == 0x1fe0000 && + observed.align == 0x10000, + "PPSA02721 Standard64KB render-target footprint changed"); - constexpr uint32_t width = 257; - constexpr uint32_t height = 131; - const auto pitch = TileGetTexturePitch(format, width, tile); - TileSizeAlign storage {}; - TileGetTextureSize(format, width, height, 1, tile, &storage, nullptr, nullptr); - Require("Standard64RenderTarget", "partial layout", - pitch == 384 && storage.size == 0x60000 && storage.align == 0x10000, - "partial Standard64KB footprint was not padded in 128x128 blocks"); + constexpr uint32_t width = 257; + constexpr uint32_t height = 131; + const auto pitch = TileGetTexturePitch(format, width, tile); + TileSizeAlign storage{}; + TileGetTextureSize(format, width, height, 1, tile, &storage, nullptr, + nullptr); + Require("Standard64RenderTarget", "partial layout", + pitch == 384 && storage.size == 0x60000 && storage.align == 0x10000, + "partial Standard64KB footprint was not padded in 128x128 blocks"); - ImageInfo info {}; - info.data = {0x10000, storage.size}; - info.pixel_format = vk::Format::eR8G8B8A8Unorm; - info.guest_format = format; - info.type = Prospero::ImageType::kColor2D; - info.extent = {width, height, 1}; - info.resources = {1, 1}; - info.pitch = pitch; - info.bytes_per_block = 4; - info.samples = 1; - info.tile_mode = tile; - info.mip_layout[0] = {0, storage.size, pitch, height}; - Require("Standard64RenderTarget", "support boundary", - IsSupportedStandard64RenderTarget(info) && IsTiledRenderTarget(info), - "exact Standard64KB render target was not classified as tiled"); - Require("Standard64RenderTarget", "display tile boundary", - IsSupportedDisplayRenderTargetTileMode( - Prospero::TileMode::kRenderTarget) && - !IsSupportedDisplayRenderTargetTileMode(tile), - "Standard64KB render target could alias a mode-27 display image"); - auto unsupported = info; - unsupported.data.address += 4; - Require("Standard64RenderTarget", "address guard", - !IsSupportedStandard64RenderTarget(unsupported), - "unaligned Standard64KB backing was accepted"); - unsupported = info; - unsupported.bytes_per_block = 8; - Require("Standard64RenderTarget", "element guard", - !IsSupportedStandard64RenderTarget(unsupported), - "unimplemented Standard64KB element size was accepted"); - unsupported = info; - unsupported.pitch += 128; - Require("Standard64RenderTarget", "pitch guard", - !IsSupportedStandard64RenderTarget(unsupported), - "non-minimal Standard64KB pitch was accepted"); - unsupported = info; - unsupported.data.size += 0x10000; - Require("Standard64RenderTarget", "size guard", !IsSupportedStandard64RenderTarget(unsupported), - "non-exact Standard64KB allocation was accepted"); - unsupported = info; - unsupported.resources.levels = 2; - Require("Standard64RenderTarget", "mip guard", !IsSupportedStandard64RenderTarget(unsupported), - "unimplemented Standard64KB mip chain was accepted"); - unsupported = info; - unsupported.resources.layers = 2; - Require("Standard64RenderTarget", "layer guard", - !IsSupportedStandard64RenderTarget(unsupported), - "unimplemented Standard64KB array was accepted"); + ImageInfo info{}; + info.data = {0x10000, storage.size}; + info.pixel_format = vk::Format::eR8G8B8A8Unorm; + info.guest_format = format; + info.type = Prospero::ImageType::kColor2D; + info.extent = {width, height, 1}; + info.resources = {1, 1}; + info.pitch = pitch; + info.bytes_per_block = 4; + info.samples = 1; + info.tile_mode = tile; + info.mip_layout[0] = {0, storage.size, pitch, height}; + Require("Standard64RenderTarget", "support boundary", + IsSupportedStandard64RenderTarget(info) && IsTiledRenderTarget(info), + "exact Standard64KB render target was not classified as tiled"); + Require("Standard64RenderTarget", "display tile boundary", + IsSupportedDisplayRenderTargetTileMode( + Prospero::TileMode::kRenderTarget) && + !IsSupportedDisplayRenderTargetTileMode(tile), + "Standard64KB render target could alias a mode-27 display image"); + auto unsupported = info; + unsupported.data.address += 4; + Require("Standard64RenderTarget", "address guard", + !IsSupportedStandard64RenderTarget(unsupported), + "unaligned Standard64KB backing was accepted"); + unsupported = info; + unsupported.bytes_per_block = 8; + Require("Standard64RenderTarget", "element guard", + !IsSupportedStandard64RenderTarget(unsupported), + "unimplemented Standard64KB element size was accepted"); + unsupported = info; + unsupported.pitch += 128; + Require("Standard64RenderTarget", "pitch guard", + !IsSupportedStandard64RenderTarget(unsupported), + "non-minimal Standard64KB pitch was accepted"); + unsupported = info; + unsupported.data.size += 0x10000; + Require("Standard64RenderTarget", "size guard", + !IsSupportedStandard64RenderTarget(unsupported), + "non-exact Standard64KB allocation was accepted"); + unsupported = info; + unsupported.resources.levels = 2; + Require("Standard64RenderTarget", "mip guard", + !IsSupportedStandard64RenderTarget(unsupported), + "unimplemented Standard64KB mip chain was accepted"); + unsupported = info; + unsupported.resources.layers = 2; + Require("Standard64RenderTarget", "layer guard", + !IsSupportedStandard64RenderTarget(unsupported), + "unimplemented Standard64KB array was accepted"); - std::printf("[host] %-32s ok\n", "Standard64RenderTarget"); + std::printf("[host] %-32s ok\n", "Standard64RenderTarget"); } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS void CheckStorageTextureGpuOwnedRebindState() { - constexpr uintptr_t base = 0x0000000200200000ull; - constexpr uint64_t size = 0x10000; - const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory( - base, size, Common::VirtualMemory::Mode::ReadWrite, "storage_texture_gpu_owned_rebind", - true); - auto* memory = reinterpret_cast(guest_memory); - Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base, - "fixed guest-owner allocation failed"); - PageManager page_manager; - MemoryTracker tracker(page_manager); - page_manager.OnGpuMap(base, size); - tracker.ForEachUploadRange( - base, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); - uint64_t readable = 0; - uint64_t mapped = 0; - MEMORY_BASIC_INFORMATION protection {}; - Require("StorageTextureGpuOwnedRebind", "owned", - tracker.IsRegionGpuModified(base, size) && - (!HostMemoryQueryReadable(base, size, readable) || readable < size) && - HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) && - mapped == size && VirtualQuery(memory, &protection, sizeof(protection)) != 0 && - protection.Protect == PAGE_NOACCESS, - "GPU-owned storage pages remained host-readable or lost tracker " - "identity"); + constexpr uintptr_t base = 0x0000000200200000ull; + constexpr uint64_t size = 0x10000; + const auto guest_memory = Libs::LibKernel::Memory::AllocateRuntimeMemory( + base, size, Common::VirtualMemory::Mode::ReadWrite, + "storage_texture_gpu_owned_rebind", true); + auto *memory = reinterpret_cast(guest_memory); + Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base, + "fixed guest-owner allocation failed"); + PageManager page_manager; + MemoryTracker tracker(page_manager); + page_manager.OnGpuMap(base, size); + tracker.ForEachUploadRange( + base, size, true, [](uint64_t, uint64_t) noexcept {}, []() noexcept {}); + uint64_t readable = 0; + uint64_t mapped = 0; + MEMORY_BASIC_INFORMATION protection{}; + Require( + "StorageTextureGpuOwnedRebind", "owned", + tracker.IsRegionGpuModified(base, size) && + (!HostMemoryQueryReadable(base, size, readable) || readable < size) && + HostMemoryQueryRange(base, size, HostMemoryAccess::Mapped, mapped) && + mapped == size && + VirtualQuery(memory, &protection, sizeof(protection)) != 0 && + protection.Protect == PAGE_NOACCESS, + "GPU-owned storage pages remained host-readable or lost tracker " + "identity"); - tracker.UnmarkRegionAsGpuModified(base, size); - readable = 0; - Require("StorageTextureGpuOwnedRebind", "clean readback", - !tracker.IsRegionGpuModified(base, size) && !tracker.IsRegionCpuModified(base, size) && - HostMemoryQueryReadable(base, size, readable) && readable == size, - "clean storage readback did not publish readable coherent backing"); - tracker.MarkRegionAsGpuModified(base, size); - readable = 0; - Require("StorageTextureGpuOwnedRebind", "clean reclaim", - tracker.IsRegionGpuModified(base, size) && !tracker.IsRegionCpuModified(base, size) && - (!HostMemoryQueryReadable(base, size, readable) || readable < size), - "clean storage rebind did not reclaim GPU ownership without an upload"); + tracker.UnmarkRegionAsGpuModified(base, size); + readable = 0; + Require("StorageTextureGpuOwnedRebind", "clean readback", + !tracker.IsRegionGpuModified(base, size) && + !tracker.IsRegionCpuModified(base, size) && + HostMemoryQueryReadable(base, size, readable) && readable == size, + "clean storage readback did not publish readable coherent backing"); + tracker.MarkRegionAsGpuModified(base, size); + readable = 0; + Require( + "StorageTextureGpuOwnedRebind", "clean reclaim", + tracker.IsRegionGpuModified(base, size) && + !tracker.IsRegionCpuModified(base, size) && + (!HostMemoryQueryReadable(base, size, readable) || readable < size), + "clean storage rebind did not reclaim GPU ownership without an upload"); - tracker.UnmarkRegionAsGpuModified(base, size); - tracker.MarkRegionAsCpuModified(base, size); - uint32_t dirty_ranges = 0; - bool upload_called = false; - tracker.ForEachUploadRange( - base, size, true, [&](uint64_t, uint64_t) noexcept { dirty_ranges++; }, - [&]() noexcept { upload_called = true; }); - readable = 0; - Require("StorageTextureGpuOwnedRebind", "dirty refresh", - dirty_ranges == 1 && upload_called && tracker.IsRegionGpuModified(base, size) && - !tracker.IsRegionCpuModified(base, size) && - (!HostMemoryQueryReadable(base, size, readable) || readable < size), - "CPU-dirty storage rebind did not refresh once and reclaim GPU " - "ownership"); + tracker.UnmarkRegionAsGpuModified(base, size); + tracker.MarkRegionAsCpuModified(base, size); + uint32_t dirty_ranges = 0; + bool upload_called = false; + tracker.ForEachUploadRange( + base, size, true, [&](uint64_t, uint64_t) noexcept { dirty_ranges++; }, + [&]() noexcept { upload_called = true; }); + readable = 0; + Require( + "StorageTextureGpuOwnedRebind", "dirty refresh", + dirty_ranges == 1 && upload_called && + tracker.IsRegionGpuModified(base, size) && + !tracker.IsRegionCpuModified(base, size) && + (!HostMemoryQueryReadable(base, size, readable) || readable < size), + "CPU-dirty storage rebind did not refresh once and reclaim GPU " + "ownership"); - tracker.UnmarkRegionAsGpuModified(base, size); - tracker.UntrackMemory(base, size); - page_manager.OnGpuUnmap(base, size); - Require("StorageTextureGpuOwnedRebind", "free", - Libs::LibKernel::Memory::FreeGuestMemory(base, size), "guest-owner free failed"); - std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind"); + tracker.UnmarkRegionAsGpuModified(base, size); + tracker.UntrackMemory(base, size); + page_manager.OnGpuUnmap(base, size); + Require("StorageTextureGpuOwnedRebind", "free", + Libs::LibKernel::Memory::FreeGuestMemory(base, size), + "guest-owner free failed"); + std::printf("[host] %-32s ok\n", "StorageTextureGpuOwnedRebind"); } #endif void CheckNativeMsaaState() { - Require("NativeMsaaState", "sample encoding", - render_sample_count(0) == 1 && render_sample_count(1) == 2 && - render_sample_count(2) == 4 && render_sample_count(3) == 8 && - render_sample_count(4) == 0, - "PS5 sample encodings were not mapped exactly"); - Require("NativeMsaaState", "Vulkan sample mapping", - vulkan_sample_count(1) == vk::SampleCountFlagBits::e1 && - vulkan_sample_count(2) == vk::SampleCountFlagBits::e2 && - vulkan_sample_count(4) == vk::SampleCountFlagBits::e4 && - vulkan_sample_count(8) == vk::SampleCountFlagBits::e8 && - vulkan_sample_count(3) == vk::SampleCountFlagBits {}, - "native sample counts were not mapped exactly to Vulkan"); + Require("NativeMsaaState", "sample encoding", + render_sample_count(0) == 1 && render_sample_count(1) == 2 && + render_sample_count(2) == 4 && render_sample_count(3) == 8 && + render_sample_count(4) == 0, + "PS5 sample encodings were not mapped exactly"); + Require("NativeMsaaState", "Vulkan sample mapping", + vulkan_sample_count(1) == vk::SampleCountFlagBits::e1 && + vulkan_sample_count(2) == vk::SampleCountFlagBits::e2 && + vulkan_sample_count(4) == vk::SampleCountFlagBits::e4 && + vulkan_sample_count(8) == vk::SampleCountFlagBits::e8 && + vulkan_sample_count(3) == vk::SampleCountFlagBits{}, + "native sample counts were not mapped exactly to Vulkan"); - TileSizeAlign color {}; - const auto color_pitch = TileGetRenderTargetPitch(1920, 8, 3); - Require("NativeMsaaState", "8x color footprint", - color_pitch == 1920 && TileGetRenderTargetSize(1920, 1080, color_pitch, 8, color, 3) && - color.align == 0x10000 && color.size == 0x07f80000, - "8x R16G16B16A16 color footprint was not preserved"); + TileSizeAlign color{}; + const auto color_pitch = TileGetRenderTargetPitch(1920, 8, 3); + Require("NativeMsaaState", "8x color footprint", + color_pitch == 1920 && + TileGetRenderTargetSize(1920, 1080, color_pitch, 8, color, 3) && + color.align == 0x10000 && color.size == 0x07f80000, + "8x R16G16B16A16 color footprint was not preserved"); - TileSizeAlign depth {}; - TileSizeAlign stencil {}; - TileSizeAlign htile {}; - Require("NativeMsaaState", "8x depth/stencil footprint", - TileGetDepthPitch(1920, 4, 3) == 1920 && - TileGetDepthSize(1920, 1080, 0, - Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, true, - stencil, htile, depth, 3) && - depth.align == 0x10000 && depth.size == 0x03fc0000 && stencil.align == 0x10000 && - stencil.size == 0x010e0000 && htile.align == 0x8000 && htile.size == 0x00030000, - "8x depth/stencil or fragment-independent HTile footprint regressed"); - std::printf("[host] %-32s ok\n", "NativeMsaaState"); + TileSizeAlign depth{}; + TileSizeAlign stencil{}; + TileSizeAlign htile{}; + Require("NativeMsaaState", "8x depth/stencil footprint", + TileGetDepthPitch(1920, 4, 3) == 1920 && + TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, true, stencil, + htile, depth, 3) && + depth.align == 0x10000 && depth.size == 0x03fc0000 && + stencil.align == 0x10000 && stencil.size == 0x010e0000 && + htile.align == 0x8000 && htile.size == 0x00030000, + "8x depth/stencil or fragment-independent HTile footprint regressed"); + std::printf("[host] %-32s ok\n", "NativeMsaaState"); } void CheckDepthHtileStencilCompatibility() { - Require("DepthHtileStencilCompatibility", "disabled acceleration", - depth_htile_stencil_acceleration_compatible(false, false, true), - "disabled Hi-Stencil state was rejected"); - Require("DepthHtileStencilCompatibility", "PS5 stencil plus HTile", - depth_htile_stencil_acceleration_compatible(true, true, false), - "valid PS5 Hi-Stencil attachment was rejected"); - Require("DepthHtileStencilCompatibility", "missing stencil plane", - !depth_htile_stencil_acceleration_compatible(false, true, false), - "Hi-Stencil without a stencil plane was silently admitted"); - Require("DepthHtileStencilCompatibility", "missing HTile metadata", - !depth_htile_stencil_acceleration_compatible(true, false, false), - "Hi-Stencil without HTile metadata was silently admitted"); - std::printf("[host] %-32s ok\n", "DepthHtileStencilCompatibility"); + Require("DepthHtileStencilCompatibility", "disabled acceleration", + depth_htile_stencil_acceleration_compatible(false, false, true), + "disabled Hi-Stencil state was rejected"); + Require("DepthHtileStencilCompatibility", "PS5 stencil plus HTile", + depth_htile_stencil_acceleration_compatible(true, true, false), + "valid PS5 Hi-Stencil attachment was rejected"); + Require("DepthHtileStencilCompatibility", "missing stencil plane", + !depth_htile_stencil_acceleration_compatible(false, true, false), + "Hi-Stencil without a stencil plane was silently admitted"); + Require("DepthHtileStencilCompatibility", "missing HTile metadata", + !depth_htile_stencil_acceleration_compatible(true, false, false), + "Hi-Stencil without HTile metadata was silently admitted"); + std::printf("[host] %-32s ok\n", "DepthHtileStencilCompatibility"); } void CheckPs5DepthRegisterDecoding() { - constexpr uint32_t captured_z = 0x22900803u; - constexpr uint32_t captured_stencil = 0x00100801u; - const auto z = HW::DepthZInfo::Decode(captured_z); - const auto stencil = HW::DepthStencilInfo::Decode(captured_stencil); - Require("Ps5DepthRegisterDecoding", "captured texture-compatible state", - z.format == Prospero::DepthFormat::kZ32F && - z.num_samples == 0 && z.htile_acceleration && !z.expclear_enabled && - !z.partially_resident && z.max_mip_level == 0 && - z.z_compare_base == Prospero::ZCompareBase::kZMin && - z.texture_compatibility == Prospero::TextureCompatiblePlaneCompression::kEnable && - z.HasValidTextureCompatibility() && - stencil.format == Prospero::StencilFormat::k8UInt && - !stencil.htile_stencil_disabled && !stencil.expclear_enabled && - !stencil.partially_resident && - stencil.texture_compatibility == Prospero::TextureCompatibleStencil::kEnable && - stencil.HasValidTextureCompatibility(), - "valid PS5 depth/stencil aggregate encodings were not decoded exactly"); + constexpr uint32_t captured_z = 0x22900803u; + constexpr uint32_t captured_stencil = 0x00100801u; + const auto z = HW::DepthZInfo::Decode(captured_z); + const auto stencil = HW::DepthStencilInfo::Decode(captured_stencil); + Require( + "Ps5DepthRegisterDecoding", "captured texture-compatible state", + z.format == Prospero::DepthFormat::kZ32F && z.num_samples == 0 && + z.htile_acceleration && !z.expclear_enabled && + !z.partially_resident && z.max_mip_level == 0 && + z.z_compare_base == Prospero::ZCompareBase::kZMin && + z.texture_compatibility == + Prospero::TextureCompatiblePlaneCompression::kEnable && + z.HasValidTextureCompatibility() && + stencil.format == Prospero::StencilFormat::k8UInt && + !stencil.htile_stencil_disabled && !stencil.expclear_enabled && + !stencil.partially_resident && + stencil.texture_compatibility == + Prospero::TextureCompatibleStencil::kEnable && + stencil.HasValidTextureCompatibility(), + "valid PS5 depth/stencil aggregate encodings were not decoded exactly"); - const auto malformed_z = HW::DepthZInfo::Decode(0x00000803u); - const auto malformed_stencil = HW::DepthStencilInfo::Decode(0x00100001u); - Require("Ps5DepthRegisterDecoding", "malformed aggregate state", - !malformed_z.HasValidTextureCompatibility() && - !malformed_stencil.HasValidTextureCompatibility(), - "partial texture-compatible aggregate encodings were accepted"); - std::printf("[host] %-32s ok\n", "Ps5DepthRegisterDecoding"); + const auto malformed_z = HW::DepthZInfo::Decode(0x00000803u); + const auto malformed_stencil = HW::DepthStencilInfo::Decode(0x00100001u); + Require("Ps5DepthRegisterDecoding", "malformed aggregate state", + !malformed_z.HasValidTextureCompatibility() && + !malformed_stencil.HasValidTextureCompatibility(), + "partial texture-compatible aggregate encodings were accepted"); + std::printf("[host] %-32s ok\n", "Ps5DepthRegisterDecoding"); } void CheckStencilAttachmentAccess() { - PipelineStencilStaticState state {vk::StencilOp::eKeep, vk::StencilOp::eKeep, - vk::StencilOp::eKeep, vk::CompareOp::eAlways}; - PipelineStencilDynamicState dynamic {0xff, 0xff, 0}; - Require("StencilAttachmentAccess", "always keep", - !stencil_face_accesses_attachment(state, dynamic), - "ALWAYS/KEEP state was classified as stencil access"); - state.compareOp = vk::CompareOp::eEqual; - Require("StencilAttachmentAccess", "compare reads", - stencil_face_accesses_attachment(state, dynamic), - "real stencil comparison was classified as no access"); - state.compareOp = vk::CompareOp::eAlways; - state.passOp = vk::StencilOp::eZero; - Require("StencilAttachmentAccess", "write operation", - stencil_face_accesses_attachment(state, dynamic), - "write-capable stencil operation was classified as no access"); - dynamic.writeMask = 0; - Require("StencilAttachmentAccess", "masked write", - !stencil_face_accesses_attachment(state, dynamic), - "fully masked stencil write was classified as access"); - std::printf("[host] %-32s ok\n", "StencilAttachmentAccess"); + PipelineStencilStaticState state{vk::StencilOp::eKeep, vk::StencilOp::eKeep, + vk::StencilOp::eKeep, + vk::CompareOp::eAlways}; + PipelineStencilDynamicState dynamic{0xff, 0xff, 0}; + Require("StencilAttachmentAccess", "always keep", + !stencil_face_accesses_attachment(state, dynamic), + "ALWAYS/KEEP state was classified as stencil access"); + state.compareOp = vk::CompareOp::eEqual; + Require("StencilAttachmentAccess", "compare reads", + stencil_face_accesses_attachment(state, dynamic), + "real stencil comparison was classified as no access"); + state.compareOp = vk::CompareOp::eAlways; + state.passOp = vk::StencilOp::eZero; + Require("StencilAttachmentAccess", "write operation", + stencil_face_accesses_attachment(state, dynamic), + "write-capable stencil operation was classified as no access"); + dynamic.writeMask = 0; + Require("StencilAttachmentAccess", "masked write", + !stencil_face_accesses_attachment(state, dynamic), + "fully masked stencil write was classified as access"); + std::printf("[host] %-32s ok\n", "StencilAttachmentAccess"); } void CheckDepthAttachmentWrites() { - RenderDepthInfo target {}; - target.format = vk::Format::eD32SfloatS8Uint; - target.depth_write_enable = true; - Require("DepthAttachmentWrites", "disabled depth test", !target.AttachmentWriteAspects(), - "disabled depth testing claimed a depth write"); + RenderDepthInfo target{}; + target.format = vk::Format::eD32SfloatS8Uint; + target.depth_write_enable = true; + Require("DepthAttachmentWrites", "disabled depth test", + !target.AttachmentWriteAspects(), + "disabled depth testing claimed a depth write"); - target.depth_test_enable = true; - target.depth_compare_op = vk::CompareOp::eLess; - Require("DepthAttachmentWrites", "depth write", - target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, - "write-enabled depth testing did not claim depth"); + target.depth_test_enable = true; + target.depth_compare_op = vk::CompareOp::eLess; + Require("DepthAttachmentWrites", "depth write", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, + "write-enabled depth testing did not claim depth"); - target.depth_test_enable = false; - target.depth_write_enable = false; - target.depth_load_clear_enable = true; - Require("DepthAttachmentWrites", "depth clear", - target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, - "depth attachment clear did not claim depth"); + target.depth_test_enable = false; + target.depth_write_enable = false; + target.depth_load_clear_enable = true; + Require("DepthAttachmentWrites", "depth clear", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eDepth, + "depth attachment clear did not claim depth"); - target.depth_load_clear_enable = false; - target.stencil_test_enable = true; - target.stencil_dynamic_front = {0xff, 0xff, 0}; - target.stencil_dynamic_back = target.stencil_dynamic_front; - target.stencil_static_front = {vk::StencilOp::eKeep, vk::StencilOp::eKeep, vk::StencilOp::eKeep, - vk::CompareOp::eAlways}; - target.stencil_static_back = target.stencil_static_front; - Require("DepthAttachmentWrites", "stencil keep", !target.AttachmentWriteAspects(), - "KEEP-only stencil state claimed a stencil write"); + target.depth_load_clear_enable = false; + target.stencil_test_enable = true; + target.stencil_dynamic_front = {0xff, 0xff, 0}; + target.stencil_dynamic_back = target.stencil_dynamic_front; + target.stencil_static_front = {vk::StencilOp::eKeep, vk::StencilOp::eKeep, + vk::StencilOp::eKeep, vk::CompareOp::eAlways}; + target.stencil_static_back = target.stencil_static_front; + Require("DepthAttachmentWrites", "stencil keep", + !target.AttachmentWriteAspects(), + "KEEP-only stencil state claimed a stencil write"); - target.stencil_static_front.failOp = vk::StencilOp::eZero; - Require("DepthAttachmentWrites", "unreachable stencil fail", !target.AttachmentWriteAspects(), - "ALWAYS comparison claimed an unreachable fail operation"); + target.stencil_static_front.failOp = vk::StencilOp::eZero; + Require("DepthAttachmentWrites", "unreachable stencil fail", + !target.AttachmentWriteAspects(), + "ALWAYS comparison claimed an unreachable fail operation"); - target.stencil_static_front.failOp = vk::StencilOp::eKeep; - target.stencil_static_front.passOp = vk::StencilOp::eReplace; - Require("DepthAttachmentWrites", "stencil pass write", - target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil, - "write-capable stencil pass did not claim stencil"); + target.stencil_static_front.failOp = vk::StencilOp::eKeep; + target.stencil_static_front.passOp = vk::StencilOp::eReplace; + Require("DepthAttachmentWrites", "stencil pass write", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil, + "write-capable stencil pass did not claim stencil"); - target.stencil_dynamic_front.writeMask = 0; - Require("DepthAttachmentWrites", "back-face keep", !target.AttachmentWriteAspects(), - "masked front write or KEEP-only back face claimed stencil"); + target.stencil_dynamic_front.writeMask = 0; + Require("DepthAttachmentWrites", "back-face keep", + !target.AttachmentWriteAspects(), + "masked front write or KEEP-only back face claimed stencil"); - target.stencil_test_enable = false; - target.stencil_clear_enable = true; - Require("DepthAttachmentWrites", "stencil clear", - target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil, - "stencil attachment clear did not claim stencil"); + target.stencil_test_enable = false; + target.stencil_clear_enable = true; + Require("DepthAttachmentWrites", "stencil clear", + target.AttachmentWriteAspects() == vk::ImageAspectFlagBits::eStencil, + "stencil attachment clear did not claim stencil"); - target.format = vk::Format::eD32Sfloat; - Require("DepthAttachmentWrites", "missing stencil aspect", !target.AttachmentWriteAspects(), - "depth-only format claimed a stencil write"); - std::printf("[host] %-32s ok\n", "DepthAttachmentWrites"); + target.format = vk::Format::eD32Sfloat; + Require("DepthAttachmentWrites", "missing stencil aspect", + !target.AttachmentWriteAspects(), + "depth-only format claimed a stencil write"); + std::printf("[host] %-32s ok\n", "DepthAttachmentWrites"); } void CheckDynamicRenderingState() { - RenderState first {}; - first.width = 64; - first.height = 32; - first.num_layers = 2; - first.num_color_attachments = 1; - first.color_attachments[0].image_layout = vk::ImageLayout::eColorAttachmentOptimal; - first.color_attachments[0].clear_value = {1, 2, 3, 4}; - first.color_attachments[0].is_clear = true; - auto changed = first; - Require("DynamicRenderingState", "identical state", first == changed, - "fieldwise RenderState identity rejected an identical scope"); - changed.color_attachments[0].is_clear = false; - Require("DynamicRenderingState", "clear identity", first != changed, - "clear/load state did not participate in rendering identity"); - changed = first; - changed.num_layers = 1; - Require("DynamicRenderingState", "layer identity", first != changed, - "layer count did not participate in rendering identity"); - changed = first; - changed.color_attachments[0].image_layout = vk::ImageLayout::eGeneral; - Require("DynamicRenderingState", "layout identity", first != changed, - "attachment layout did not participate in rendering identity"); + RenderState first{}; + first.width = 64; + first.height = 32; + first.num_layers = 2; + first.num_color_attachments = 1; + first.color_attachments[0].image_layout = + vk::ImageLayout::eColorAttachmentOptimal; + first.color_attachments[0].clear_value = {1, 2, 3, 4}; + first.color_attachments[0].is_clear = true; + auto changed = first; + Require("DynamicRenderingState", "identical state", first == changed, + "fieldwise RenderState identity rejected an identical scope"); + changed.color_attachments[0].is_clear = false; + Require("DynamicRenderingState", "clear identity", first != changed, + "clear/load state did not participate in rendering identity"); + changed = first; + changed.num_layers = 1; + Require("DynamicRenderingState", "layer identity", first != changed, + "layer count did not participate in rendering identity"); + changed = first; + changed.color_attachments[0].image_layout = vk::ImageLayout::eGeneral; + Require("DynamicRenderingState", "layout identity", first != changed, + "attachment layout did not participate in rendering identity"); - PipelineRenderingState rgba {}; - rgba.color_count = 1; - rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm; - auto uint_color = rgba; - uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint; - Require("DynamicRenderingState", "pipeline color format identity", rgba != uint_color, - "distinct dynamic-rendering formats can alias one pipeline key"); - auto depth = rgba; - depth.depth_format = vk::Format::eD32SfloatS8Uint; - depth.stencil_format = vk::Format::eD32SfloatS8Uint; - Require("DynamicRenderingState", "pipeline depth/stencil identity", rgba != depth, - "depth/stencil formats did not participate in pipeline identity"); + PipelineRenderingState rgba{}; + rgba.color_count = 1; + rgba.color_formats[0] = vk::Format::eR8G8B8A8Unorm; + auto uint_color = rgba; + uint_color.color_formats[0] = vk::Format::eR8G8B8A8Uint; + Require("DynamicRenderingState", "pipeline color format identity", + rgba != uint_color, + "distinct dynamic-rendering formats can alias one pipeline key"); + auto depth = rgba; + depth.depth_format = vk::Format::eD32SfloatS8Uint; + depth.stencil_format = vk::Format::eD32SfloatS8Uint; + Require("DynamicRenderingState", "pipeline depth/stencil identity", + rgba != depth, + "depth/stencil formats did not participate in pipeline identity"); - RenderDepthInfo attachment {}; - attachment.format = vk::Format::eD32SfloatS8Uint; - Require("DynamicRenderingState", "read-only depth/stencil layout", - depth_attachment_layout(attachment) == vk::ImageLayout::eDepthStencilReadOnlyOptimal, - "fully read-only depth/stencil used a writable layout"); - attachment.depth_load_clear_enable = true; - Require("DynamicRenderingState", "depth-write stencil-read layout", - depth_attachment_layout(attachment) == - vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal, - "depth-only writes did not retain read-only stencil"); - attachment.depth_load_clear_enable = false; - attachment.stencil_clear_enable = true; - Require("DynamicRenderingState", "depth-read stencil-write layout", - depth_attachment_layout(attachment) == - vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal, - "stencil-only writes did not retain read-only depth"); - attachment.depth_load_clear_enable = true; - Require("DynamicRenderingState", "writable depth/stencil layout", - depth_attachment_layout(attachment) == vk::ImageLayout::eDepthStencilAttachmentOptimal, - "combined depth/stencil writes did not use the writable layout"); - std::printf("[host] %-32s ok\n", "DynamicRenderingState"); + RenderDepthInfo attachment{}; + attachment.format = vk::Format::eD32SfloatS8Uint; + Require("DynamicRenderingState", "read-only depth/stencil layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthStencilReadOnlyOptimal, + "fully read-only depth/stencil used a writable layout"); + attachment.depth_load_clear_enable = true; + Require("DynamicRenderingState", "depth-write stencil-read layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthAttachmentStencilReadOnlyOptimal, + "depth-only writes did not retain read-only stencil"); + attachment.depth_load_clear_enable = false; + attachment.stencil_clear_enable = true; + Require("DynamicRenderingState", "depth-read stencil-write layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthReadOnlyStencilAttachmentOptimal, + "stencil-only writes did not retain read-only depth"); + attachment.depth_load_clear_enable = true; + Require("DynamicRenderingState", "writable depth/stencil layout", + depth_attachment_layout(attachment) == + vk::ImageLayout::eDepthStencilAttachmentOptimal, + "combined depth/stencil writes did not use the writable layout"); + std::printf("[host] %-32s ok\n", "DynamicRenderingState"); } void CheckDepthTargetFootprints() { - TileSizeAlign stencil {}; - TileSizeAlign htile {}; - TileSizeAlign depth {}; - struct AttachmentFormatCase { - const char* name; - Prospero::DepthFormat depth_format; - Prospero::StencilFormat stencil_format; - vk::Format expected; - }; - constexpr AttachmentFormatCase attachment_cases[] = { - {"Z16", Prospero::DepthFormat::kZ16, - Prospero::StencilFormat::kInvalid, vk::Format::eD16Unorm}, - {"Z16S8", Prospero::DepthFormat::kZ16, - Prospero::StencilFormat::k8UInt, vk::Format::eD16UnormS8Uint}, - {"Z32", Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::kInvalid, vk::Format::eD32Sfloat}, - {"Z32S8", Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, vk::Format::eD32SfloatS8Uint}, - {"invalid depth", static_cast(2u), - Prospero::StencilFormat::k8UInt, - vk::Format::eUndefined}, - {"invalid stencil", Prospero::DepthFormat::kZ16, - static_cast(2u), - vk::Format::eUndefined}, - }; - for (const auto& test: attachment_cases) { - Require("DepthTargetFootprints", test.name, - DepthAttachmentFormat(test.depth_format, test.stencil_format) == test.expected, - "PS5 depth/stencil attachment mapping changed"); - } - Require("DepthTargetFootprints", "1920x1080 Z16S8 without HTile", - TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ16, - Prospero::StencilFormat::k8UInt, false, - stencil, htile, depth) && - depth.size == 0x480000 && depth.align == 0x10000 && stencil.size == 0x280000 && - stencil.align == 0x10000 && htile.size == 0, - "captured Z16S8 footprint disagrees with Prospero block rules"); - struct TargetFormatCase { - const char* name; - vk::Format host_format; - Prospero::BufferFormat guest_format; - uint32_t bytes_per_element; - bool stencil; - bool supported; - bool readback; - }; - constexpr TargetFormatCase target_cases[] = { - {"D16", vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm, 2, - false, true, true}, - {"D16S8", vk::Format::eD16UnormS8Uint, - Prospero::BufferFormat::k16UNorm, 2, true, true, true}, - {"D16 via D24S8", vk::Format::eD24UnormS8Uint, - Prospero::BufferFormat::k16UNorm, 2, true, true, true}, - {"D16 via D32S8", vk::Format::eD32SfloatS8Uint, - Prospero::BufferFormat::k16UNorm, 2, true, true, true}, - {"D32", vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, 4, - false, true, true}, - {"D32S8", vk::Format::eD32SfloatS8Uint, - Prospero::BufferFormat::k32Float, 4, true, true, true}, - {"D16 plus stencil mismatch", vk::Format::eD16Unorm, - Prospero::BufferFormat::k16UNorm, 2, true, false, false}, - {"fallback without stencil", vk::Format::eD24UnormS8Uint, - Prospero::BufferFormat::k16UNorm, 2, false, false, false}, - {"D32 guest via D24", vk::Format::eD24UnormS8Uint, - Prospero::BufferFormat::k32Float, 4, true, false, false}, - {"D16 byte mismatch", vk::Format::eD16Unorm, - Prospero::BufferFormat::k16UNorm, 4, false, false, false}, - }; - for (const auto& test: target_cases) { - ImageInfo target {}; - target.pixel_format = test.host_format; - target.guest_format = test.guest_format; - target.bytes_per_block = test.bytes_per_element; - target.stencil = test.stencil ? GuestRange {0x10000, 0x10000} : GuestRange {}; - Require("DepthTargetFootprints", test.name, - IsSupportedDepthTargetFormat(target) == test.supported && - IsSupportedDepthPlaneReadback(target) == test.readback, - "host/guest depth format or exact readback policy changed"); - } - ImageInfo compressed_stencil {}; - compressed_stencil.pixel_format = vk::Format::eD32SfloatS8Uint; - compressed_stencil.guest_format = Prospero::BufferFormat::k32Float; - compressed_stencil.bytes_per_block = 4; - compressed_stencil.stencil = {0x10000, 0x10000}; - compressed_stencil.metadata.stencil_compressed = true; - Require("DepthTargetFootprints", "compressed stencil depth-plane readback", - IsSupportedDepthTargetFormat(compressed_stencil) && - IsSupportedDepthPlaneReadback(compressed_stencil), - "stencil metadata incorrectly blocked independent depth-plane " - "readback"); - Require("DepthTargetFootprints", "lexicographic subresource comparison", - ImageSubresources {2, 1} > ImageSubresources {1, 4} && - ImageSubresources {1, 4} < ImageSubresources {2, 1}, - "subresource ordering diverged from required overlap semantics"); - struct TransferPlaneCase { - const char* name; - vk::Format attachment; - vk::Format transfer; - uint32_t bytes; - }; - constexpr TransferPlaneCase transfer_cases[] = { - {"D16S8 transfer", vk::Format::eD16UnormS8Uint, vk::Format::eD16Unorm, 2}, - {"D24S8 transfer", vk::Format::eD24UnormS8Uint, vk::Format::eX8D24UnormPack32, 4}, - {"D32S8 transfer", vk::Format::eD32SfloatS8Uint, vk::Format::eD32Sfloat, 4}, - {"invalid transfer", vk::Format::eR16Unorm, vk::Format::eUndefined, 0}, - }; - for (const auto& test: transfer_cases) { - Require("DepthTargetFootprints", test.name, - DepthAspectTransferFormat(test.attachment) == test.transfer && - DepthAspectTransferBytes(test.attachment) == test.bytes, - "combined depth transfer-plane layout changed"); - } - struct PromotionCase { - const char* name; - uint16_t source; - uint32_t d24; - uint32_t d32; - }; - constexpr PromotionCase promotion_cases[] = { - {"zero", 0, 0, 0}, - {"midpoint", 0x8000, 0x00800080, 0x3f000080}, - {"maximum", 0xffff, 0x00ffffff, 0x3f800000}, - }; - for (const auto& test: promotion_cases) { - Require("DepthTargetFootprints", test.name, - EncodeD16AsD24(test.source) == test.d24 && EncodeD16AsD32(test.source) == test.d32, - "D16 host promotion changed the represented depth value"); - } - Require("DepthTargetFootprints", "640x360 Z32S8 without HTile", - TileGetDepthSize(640, 360, 0, Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, false, - stencil, htile, depth), - "valid non-HTile depth/stencil footprint was rejected"); - Require("DepthTargetFootprints", "640x360 Prospero block sizes", - depth.size == 0xf0000 && depth.align == 0x10000 && stencil.size == 0x60000 && - stencil.align == 0x10000 && htile.size == 0 && htile.align == 0, - "non-HTile depth/stencil footprint disagrees with Prospero block rules"); + TileSizeAlign stencil{}; + TileSizeAlign htile{}; + TileSizeAlign depth{}; + struct AttachmentFormatCase { + const char *name; + Prospero::DepthFormat depth_format; + Prospero::StencilFormat stencil_format; + vk::Format expected; + }; + constexpr AttachmentFormatCase attachment_cases[] = { + {"Z16", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::kInvalid, + vk::Format::eD16Unorm}, + {"Z16S8", Prospero::DepthFormat::kZ16, Prospero::StencilFormat::k8UInt, + vk::Format::eD16UnormS8Uint}, + {"Z32", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::kInvalid, + vk::Format::eD32Sfloat}, + {"Z32S8", Prospero::DepthFormat::kZ32F, Prospero::StencilFormat::k8UInt, + vk::Format::eD32SfloatS8Uint}, + {"invalid depth", static_cast(2u), + Prospero::StencilFormat::k8UInt, vk::Format::eUndefined}, + {"invalid stencil", Prospero::DepthFormat::kZ16, + static_cast(2u), vk::Format::eUndefined}, + }; + for (const auto &test : attachment_cases) { + Require("DepthTargetFootprints", test.name, + DepthAttachmentFormat(test.depth_format, test.stencil_format) == + test.expected, + "PS5 depth/stencil attachment mapping changed"); + } + Require("DepthTargetFootprints", "1920x1080 Z16S8 without HTile", + TileGetDepthSize(1920, 1080, 0, Prospero::DepthFormat::kZ16, + Prospero::StencilFormat::k8UInt, false, stencil, + htile, depth) && + depth.size == 0x480000 && depth.align == 0x10000 && + stencil.size == 0x280000 && stencil.align == 0x10000 && + htile.size == 0, + "captured Z16S8 footprint disagrees with Prospero block rules"); + struct TargetFormatCase { + const char *name; + vk::Format host_format; + Prospero::BufferFormat guest_format; + uint32_t bytes_per_element; + bool stencil; + bool supported; + bool readback; + }; + constexpr TargetFormatCase target_cases[] = { + {"D16", vk::Format::eD16Unorm, Prospero::BufferFormat::k16UNorm, 2, false, + true, true}, + {"D16S8", vk::Format::eD16UnormS8Uint, Prospero::BufferFormat::k16UNorm, + 2, true, true, true}, + {"D16 via D24S8", vk::Format::eD24UnormS8Uint, + Prospero::BufferFormat::k16UNorm, 2, true, true, true}, + {"D16 via D32S8", vk::Format::eD32SfloatS8Uint, + Prospero::BufferFormat::k16UNorm, 2, true, true, true}, + {"D32", vk::Format::eD32Sfloat, Prospero::BufferFormat::k32Float, 4, + false, true, true}, + {"D32S8", vk::Format::eD32SfloatS8Uint, Prospero::BufferFormat::k32Float, + 4, true, true, true}, + {"D16 plus stencil mismatch", vk::Format::eD16Unorm, + Prospero::BufferFormat::k16UNorm, 2, true, false, false}, + {"fallback without stencil", vk::Format::eD24UnormS8Uint, + Prospero::BufferFormat::k16UNorm, 2, false, false, false}, + {"D32 guest via D24", vk::Format::eD24UnormS8Uint, + Prospero::BufferFormat::k32Float, 4, true, false, false}, + {"D16 byte mismatch", vk::Format::eD16Unorm, + Prospero::BufferFormat::k16UNorm, 4, false, false, false}, + }; + for (const auto &test : target_cases) { + ImageInfo target{}; + target.pixel_format = test.host_format; + target.guest_format = test.guest_format; + target.bytes_per_block = test.bytes_per_element; + target.stencil = test.stencil ? GuestRange{0x10000, 0x10000} : GuestRange{}; + Require("DepthTargetFootprints", test.name, + IsSupportedDepthTargetFormat(target) == test.supported && + IsSupportedDepthPlaneReadback(target) == test.readback, + "host/guest depth format or exact readback policy changed"); + } + ImageInfo compressed_stencil{}; + compressed_stencil.pixel_format = vk::Format::eD32SfloatS8Uint; + compressed_stencil.guest_format = Prospero::BufferFormat::k32Float; + compressed_stencil.bytes_per_block = 4; + compressed_stencil.stencil = {0x10000, 0x10000}; + compressed_stencil.metadata.stencil_compressed = true; + Require("DepthTargetFootprints", "compressed stencil depth-plane readback", + IsSupportedDepthTargetFormat(compressed_stencil) && + IsSupportedDepthPlaneReadback(compressed_stencil), + "stencil metadata incorrectly blocked independent depth-plane " + "readback"); + Require("DepthTargetFootprints", "lexicographic subresource comparison", + ImageSubresources{2, 1} > ImageSubresources{1, 4} && + ImageSubresources{1, 4} < ImageSubresources{2, 1}, + "subresource ordering diverged from required overlap semantics"); + struct TransferPlaneCase { + const char *name; + vk::Format attachment; + vk::Format transfer; + uint32_t bytes; + }; + constexpr TransferPlaneCase transfer_cases[] = { + {"D16S8 transfer", vk::Format::eD16UnormS8Uint, vk::Format::eD16Unorm, 2}, + {"D24S8 transfer", vk::Format::eD24UnormS8Uint, + vk::Format::eX8D24UnormPack32, 4}, + {"D32S8 transfer", vk::Format::eD32SfloatS8Uint, vk::Format::eD32Sfloat, + 4}, + {"invalid transfer", vk::Format::eR16Unorm, vk::Format::eUndefined, 0}, + }; + for (const auto &test : transfer_cases) { + Require("DepthTargetFootprints", test.name, + DepthAspectTransferFormat(test.attachment) == test.transfer && + DepthAspectTransferBytes(test.attachment) == test.bytes, + "combined depth transfer-plane layout changed"); + } + struct PromotionCase { + const char *name; + uint16_t source; + uint32_t d24; + uint32_t d32; + }; + constexpr PromotionCase promotion_cases[] = { + {"zero", 0, 0, 0}, + {"midpoint", 0x8000, 0x00800080, 0x3f000080}, + {"maximum", 0xffff, 0x00ffffff, 0x3f800000}, + }; + for (const auto &test : promotion_cases) { + Require("DepthTargetFootprints", test.name, + EncodeD16AsD24(test.source) == test.d24 && + EncodeD16AsD32(test.source) == test.d32, + "D16 host promotion changed the represented depth value"); + } + Require("DepthTargetFootprints", "640x360 Z32S8 without HTile", + TileGetDepthSize(640, 360, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, false, stencil, + htile, depth), + "valid non-HTile depth/stencil footprint was rejected"); + Require( + "DepthTargetFootprints", "640x360 Prospero block sizes", + depth.size == 0xf0000 && depth.align == 0x10000 && + stencil.size == 0x60000 && stencil.align == 0x10000 && + htile.size == 0 && htile.align == 0, + "non-HTile depth/stencil footprint disagrees with Prospero block rules"); - const auto depth_pitch = - TileGetTexturePitch(Prospero::BufferFormat::k32Float, 640, - Prospero::TileMode::kDepth); - TileSizeAlign texture_depth {}; - TileGetTextureTotalSize(Prospero::BufferFormat::k32Float, 640, 360, 1, - 1, Prospero::TileMode::kDepth, - false, texture_depth); - Require("DepthTargetFootprints", "640x360 generic depth tile", - depth_pitch == 640 && texture_depth.size == 0xf0000 && texture_depth.align == 0x10000, - "generic depth texture sizing bypassed 64 KiB block padding"); + const auto depth_pitch = TileGetTexturePitch(Prospero::BufferFormat::k32Float, + 640, Prospero::TileMode::kDepth); + TileSizeAlign texture_depth{}; + TileGetTextureTotalSize(Prospero::BufferFormat::k32Float, 640, 360, 1, 1, + Prospero::TileMode::kDepth, false, texture_depth); + Require("DepthTargetFootprints", "640x360 generic depth tile", + depth_pitch == 640 && texture_depth.size == 0xf0000 && + texture_depth.align == 0x10000, + "generic depth texture sizing bypassed 64 KiB block padding"); - Require("DepthTargetFootprints", "960x540 Z32S8 with HTile", - TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, true, stencil, - htile, depth) && - depth.size == 0x280000 && depth.align == 0x10000 && stencil.size == 0xc0000 && - stencil.align == 0x10000 && htile.size == 0x10000 && htile.align == 0x8000, - "generic Prospero HTile block calculation rejected the title footprint"); - Require("DepthTargetFootprints", "known HTile extent", - TileGetDepthSize(1280, 720, 0, Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, true, stencil, - htile, depth) && - depth.size == 0x3c0000 && stencil.size == 0xf0000 && htile.size == 0x20000, - "validated 1280x720 HTile footprint regressed"); - Require("DepthTargetFootprints", "PPSA06228 3840x2160 Z32S8 with HTile", - TileGetDepthSize(3840, 2160, 0, Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, true, stencil, - htile, depth) && - depth.size == 0x1fe0000 && depth.align == 0x10000 && stencil.size == 0x870000 && - stencil.align == 0x10000 && htile.size == 0xa0000 && htile.align == 0x8000, - "captured 3840x2160 depth/stencil/HTile footprint disagrees with " - "Prospero rules"); - Require("DepthTargetFootprints", "invalid depth format", - !TileGetDepthSize(960, 540, 0, static_cast(2u), - Prospero::StencilFormat::k8UInt, true, - stencil, htile, depth), - "unsupported depth format was silently admitted"); - Require("DepthTargetFootprints", "invalid stencil format", - !TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F, - static_cast(2u), - true, stencil, htile, depth), - "unsupported stencil format was silently admitted"); - Require("DepthTargetFootprints", "invalid extent", - !TileGetDepthSize(0, 540, 0, Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, true, - stencil, htile, depth) && - !TileGetDepthSize(16385, 540, 0, - Prospero::DepthFormat::kZ32F, - Prospero::StencilFormat::k8UInt, true, - stencil, htile, depth), - "invalid HTile extent was silently admitted"); - std::printf("[host] %-32s ok\n", "DepthTargetFootprints"); + Require( + "DepthTargetFootprints", "960x540 Z32S8 with HTile", + TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, true, stencil, htile, + depth) && + depth.size == 0x280000 && depth.align == 0x10000 && + stencil.size == 0xc0000 && stencil.align == 0x10000 && + htile.size == 0x10000 && htile.align == 0x8000, + "generic Prospero HTile block calculation rejected the title footprint"); + Require("DepthTargetFootprints", "known HTile extent", + TileGetDepthSize(1280, 720, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, true, stencil, + htile, depth) && + depth.size == 0x3c0000 && stencil.size == 0xf0000 && + htile.size == 0x20000, + "validated 1280x720 HTile footprint regressed"); + Require("DepthTargetFootprints", "PPSA06228 3840x2160 Z32S8 with HTile", + TileGetDepthSize(3840, 2160, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, true, stencil, + htile, depth) && + depth.size == 0x1fe0000 && depth.align == 0x10000 && + stencil.size == 0x870000 && stencil.align == 0x10000 && + htile.size == 0xa0000 && htile.align == 0x8000, + "captured 3840x2160 depth/stencil/HTile footprint disagrees with " + "Prospero rules"); + Require("DepthTargetFootprints", "invalid depth format", + !TileGetDepthSize(960, 540, 0, static_cast(2u), + Prospero::StencilFormat::k8UInt, true, stencil, + htile, depth), + "unsupported depth format was silently admitted"); + Require("DepthTargetFootprints", "invalid stencil format", + !TileGetDepthSize(960, 540, 0, Prospero::DepthFormat::kZ32F, + static_cast(2u), true, + stencil, htile, depth), + "unsupported stencil format was silently admitted"); + Require("DepthTargetFootprints", "invalid extent", + !TileGetDepthSize(0, 540, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, true, stencil, + htile, depth) && + !TileGetDepthSize(16385, 540, 0, Prospero::DepthFormat::kZ32F, + Prospero::StencilFormat::k8UInt, true, stencil, + htile, depth), + "invalid HTile extent was silently admitted"); + std::printf("[host] %-32s ok\n", "DepthTargetFootprints"); } struct FenceLifetimeProbe { - explicit FenceLifetimeProbe(bool* destroyed): destroyed(destroyed) { - if (destroyed == nullptr || *destroyed) { - EXIT("fence-lifetime probe has invalid construction state\n"); - } - } - ~FenceLifetimeProbe() { *destroyed = true; } + explicit FenceLifetimeProbe(bool *destroyed) : destroyed(destroyed) { + if (destroyed == nullptr || *destroyed) { + EXIT("fence-lifetime probe has invalid construction state\n"); + } + } + ~FenceLifetimeProbe() { *destroyed = true; } - bool* destroyed = nullptr; + bool *destroyed = nullptr; }; void CheckSharedFenceResourceLifetime() { - bool destroyed = false; - auto image = std::make_shared(&destroyed); - FenceResourceRetainer first; - FenceResourceRetainer second; - first.Retain(image); - second.Retain(image); - first.Retain(image); - image.reset(); - Require("SharedFenceResourceLifetime", "retained", - !destroyed && !first.Empty() && !second.Empty(), - "cache removal destroyed an image retained by command buffers"); - first.ReleaseAfterFence(); - Require("SharedFenceResourceLifetime", "first fence", - !destroyed && first.Empty() && !second.Empty(), - "first command-buffer fence destroyed another buffer's image"); - second.ReleaseAfterFence(); - Require("SharedFenceResourceLifetime", "last fence", destroyed && second.Empty(), - "last referencing command-buffer fence did not destroy the image"); - std::printf("[host] %-32s ok\n", "SharedFenceResourceLifetime"); + bool destroyed = false; + auto image = std::make_shared(&destroyed); + FenceResourceRetainer first; + FenceResourceRetainer second; + first.Retain(image); + second.Retain(image); + first.Retain(image); + image.reset(); + Require("SharedFenceResourceLifetime", "retained", + !destroyed && !first.Empty() && !second.Empty(), + "cache removal destroyed an image retained by command buffers"); + first.ReleaseAfterFence(); + Require("SharedFenceResourceLifetime", "first fence", + !destroyed && first.Empty() && !second.Empty(), + "first command-buffer fence destroyed another buffer's image"); + second.ReleaseAfterFence(); + Require("SharedFenceResourceLifetime", "last fence", + destroyed && second.Empty(), + "last referencing command-buffer fence did not destroy the image"); + std::printf("[host] %-32s ok\n", "SharedFenceResourceLifetime"); } void CheckEmbeddedFetchLaneSpill() { - std::vector code; - code.push_back(EncodeSMovB32(0, InlineU32(0))); - code.push_back(EncodeSmem0(0x02u, 20, 4)); - code.push_back(EncodeSmem1(0)); // s_load_dwordx4 s[20:23], s[8:9] - AppendVop3(&code, 0x361u, 6, 20, InlineU32(0)); // v_writelane_b32 v6, s20, 0 - code.push_back(EncodeSMovB32(20, InlineU32(0))); - AppendVop3(&code, 0x360u, 20, Vgpr(6), - InlineU32(0)); // v_readlane_b32 s20, v6, 0 - code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5)); - code.push_back(EncodeMubuf0(0x03u, 0, true)); - code.push_back(EncodeMubuf1(9, 5, 0)); - AppendEnd(&code); + std::vector code; + code.push_back(EncodeSMovB32(0, InlineU32(0))); + code.push_back(EncodeSmem0(0x02u, 20, 4)); + code.push_back(EncodeSmem1(0)); // s_load_dwordx4 s[20:23], s[8:9] + AppendVop3(&code, 0x361u, 6, 20, InlineU32(0)); // v_writelane_b32 v6, s20, 0 + code.push_back(EncodeSMovB32(20, InlineU32(0))); + AppendVop3(&code, 0x360u, 20, Vgpr(6), + InlineU32(0)); // v_readlane_b32 s20, v6, 0 + code.push_back(EncodeVop2(0x01u, 0, Vgpr(8), 5)); + code.push_back(EncodeMubuf0(0x03u, 0, true)); + code.push_back(EncodeMubuf1(9, 5, 0)); + AppendEnd(&code); - std::array user_data {}; - ShaderVertexInputInfo vertex; - vertex.fetch_embedded = true; - vertex.fetch_buffer_reg = 0; - vertex.resources_num = 1; - vertex.resources_dst[0].attr_id = 0; - vertex.resources_dst[0].registers_num = 4; + std::array user_data{}; + ShaderVertexInputInfo vertex; + vertex.fetch_embedded = true; + vertex.fetch_buffer_reg = 0; + vertex.resources_num = 1; + vertex.resources_dst[0].attr_id = 0; + vertex.resources_dst[0].registers_num = 4; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.user_data_base = 8; - options.user_data_count = static_cast(user_data.size()); - options.user_data = user_data.data(); - options.vertex_input_info = &vertex; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.user_data_base = 8; + options.user_data_count = static_cast(user_data.size()); + options.user_data = user_data.data(); + options.vertex_input_info = &vertex; - ShaderRecompiler::CompileResult result; - std::string error; - Require("EmbeddedFetchLaneSpill", "compile", - ShaderRecompiler::TryRecompile(code, options, result, &error), error); - Require("EmbeddedFetchLaneSpill", "fetch rewrite", vertex.resource_fetch_components[0] == 4, - "lane-spilled fetch descriptor was not recognized and rewritten"); - std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill"); + ShaderRecompiler::CompileResult result; + std::string error; + Require("EmbeddedFetchLaneSpill", "compile", + ShaderRecompiler::TryRecompile(code, options, result, &error), error); + Require("EmbeddedFetchLaneSpill", "fetch rewrite", + vertex.resource_fetch_components[0] == 4, + "lane-spilled fetch descriptor was not recognized and rewritten"); + std::printf("[host] %-32s ok\n", "EmbeddedFetchLaneSpill"); } void CheckReferenceClockScale() { - uint64_t value = 0; - Require("ReferenceClockScale", "zero", - Sync::ScaleReferenceClock(0, 3000000000ull, value) && value == 0, - "zero host tick did not produce a zero GPU clock"); - Require("ReferenceClockScale", "fractional second", - Sync::ScaleReferenceClock(1500000000ull, 3000000000ull, value) && value == 50000000ull, - "host half-second did not scale to 50,000,000 ticks"); - Require("ReferenceClockScale", "whole and fractional", - Sync::ScaleReferenceClock(3750000000ull, 3000000000ull, value) && value == 125000000ull, - "host 1.25 seconds did not scale to 125,000,000 ticks"); - Require("ReferenceClockScale", "monotonic floor", - Sync::ScaleReferenceClock(3750000001ull, 3000000000ull, value) && value == 125000000ull, - "sub-reference-tick increment did not use a monotonic floor"); - Require("ReferenceClockScale", "guards", - !Sync::ScaleReferenceClock(1, 0, value) && - !Sync::ScaleReferenceClock(UINT64_MAX, 1, value), - "invalid frequency or overflow was accepted"); - std::printf("[host] %-32s ok\n", "ReferenceClockScale"); + uint64_t value = 0; + Require("ReferenceClockScale", "zero", + Sync::ScaleReferenceClock(0, 3000000000ull, value) && value == 0, + "zero host tick did not produce a zero GPU clock"); + Require("ReferenceClockScale", "fractional second", + Sync::ScaleReferenceClock(1500000000ull, 3000000000ull, value) && + value == 50000000ull, + "host half-second did not scale to 50,000,000 ticks"); + Require("ReferenceClockScale", "whole and fractional", + Sync::ScaleReferenceClock(3750000000ull, 3000000000ull, value) && + value == 125000000ull, + "host 1.25 seconds did not scale to 125,000,000 ticks"); + Require("ReferenceClockScale", "monotonic floor", + Sync::ScaleReferenceClock(3750000001ull, 3000000000ull, value) && + value == 125000000ull, + "sub-reference-tick increment did not use a monotonic floor"); + Require("ReferenceClockScale", "guards", + !Sync::ScaleReferenceClock(1, 0, value) && + !Sync::ScaleReferenceClock(UINT64_MAX, 1, value), + "invalid frequency or overflow was accepted"); + std::printf("[host] %-32s ok\n", "ReferenceClockScale"); } void CheckClipControlDepthClipState() { - HW::ClipControl clip; - Require("ClipControlDepthClipState", "default", - clip.IsZClipModeRepresentable() && clip.IsZClipEnabled(), - "default paired Z clipping was not enabled"); + HW::ClipControl clip; + Require("ClipControlDepthClipState", "default", + clip.IsZClipModeRepresentable() && clip.IsZClipEnabled(), + "default paired Z clipping was not enabled"); - clip.min_z_clip_disable = true; - Require("ClipControlDepthClipState", "asymmetric near", !clip.IsZClipModeRepresentable(), - "asymmetric near-plane state was accepted"); + clip.min_z_clip_disable = true; + Require("ClipControlDepthClipState", "asymmetric near", + !clip.IsZClipModeRepresentable(), + "asymmetric near-plane state was accepted"); - clip.min_z_clip_disable = false; - clip.max_z_clip_disable = true; - Require("ClipControlDepthClipState", "asymmetric far", !clip.IsZClipModeRepresentable(), - "asymmetric far-plane state was accepted"); + clip.min_z_clip_disable = false; + clip.max_z_clip_disable = true; + Require("ClipControlDepthClipState", "asymmetric far", + !clip.IsZClipModeRepresentable(), + "asymmetric far-plane state was accepted"); - clip.min_z_clip_disable = true; - Require("ClipControlDepthClipState", "both disabled", - clip.IsZClipModeRepresentable() && !clip.IsZClipEnabled(), - "paired Z-clip disable was not represented"); - std::printf("[host] %-32s ok\n", "ClipControlDepthClipState"); + clip.min_z_clip_disable = true; + Require("ClipControlDepthClipState", "both disabled", + clip.IsZClipModeRepresentable() && !clip.IsZClipEnabled(), + "paired Z-clip disable was not represented"); + std::printf("[host] %-32s ok\n", "ClipControlDepthClipState"); } void CheckVulkan13FeatureRequirements() { - const auto features = WindowContext::RequiredVulkan13Features(); - Require("Vulkan13FeatureRequirements", "production requirements", - features.sType == vk::StructureType::ePhysicalDeviceVulkan13Features && - features.pNext == nullptr && features.dynamicRendering == VK_TRUE && - features.synchronization2 == VK_TRUE, - "production device creation did not require dynamic rendering and " - "synchronization2 from one feature declaration"); - std::printf("[host] %-32s ok\n", "Vulkan13FeatureRequirements"); + const auto features = WindowContext::RequiredVulkan13Features(); + Require( + "Vulkan13FeatureRequirements", "production requirements", + features.sType == vk::StructureType::ePhysicalDeviceVulkan13Features && + features.pNext == nullptr && features.dynamicRendering == VK_TRUE && + features.synchronization2 == VK_TRUE, + "production device creation did not require dynamic rendering and " + "synchronization2 from one feature declaration"); + std::printf("[host] %-32s ok\n", "Vulkan13FeatureRequirements"); } -void CheckPm4AcquireMemNoOp(RenderContext& renderer) { - GraphicsInitJmpTables(); - CommandProcessor processor(renderer); - const std::array standard_payload {0x00400000u, 1u, 0u, 0u, 0u, 10u}; - const std::array custom_payload { - 0xdeadbeefu, 0xffffffffu, 0x80000000u, 0x13579bdfu, 0x2468ace0u, 0xaaaaaaaau, 0x55555555u}; - const auto standard_before = standard_payload; - const auto custom_before = custom_payload; - Require("Pm4AcquireMemNoOp", "recognized packets", - CpOpAcquireMem(processor, 0xC0055800u, standard_payload.data(), 0, 0) == 6 && - CpOpAcquireMem(processor, 0xc0061050u, custom_payload.data(), 0, 0) == 7 && - standard_payload == standard_before && custom_payload == custom_before, - "ACQUIRE_MEM did not consume both pinned packet forms as " - "side-effect-free " - "no-ops"); - std::printf("[host] %-32s ok\n", "Pm4AcquireMemNoOp"); +void CheckPm4AcquireMemNoOp(RenderContext &renderer) { + GraphicsInitJmpTables(); + CommandProcessor processor(renderer); + const std::array standard_payload{0x00400000u, 1u, 0u, + 0u, 0u, 10u}; + const std::array custom_payload{ + 0xdeadbeefu, 0xffffffffu, 0x80000000u, 0x13579bdfu, + 0x2468ace0u, 0xaaaaaaaau, 0x55555555u}; + const auto standard_before = standard_payload; + const auto custom_before = custom_payload; + Require("Pm4AcquireMemNoOp", "recognized packets", + CpOpAcquireMem(processor, 0xC0055800u, standard_payload.data(), 0, + 0) == 6 && + CpOpAcquireMem(processor, 0xc0061050u, custom_payload.data(), 0, + 0) == 7 && + standard_payload == standard_before && + custom_payload == custom_before, + "ACQUIRE_MEM did not consume both pinned packet forms as " + "side-effect-free " + "no-ops"); + std::printf("[host] %-32s ok\n", "Pm4AcquireMemNoOp"); } -void CheckPm4StencilInfoValueLane(RenderContext& renderer) { - CommandProcessor processor(renderer); - constexpr std::array payload {0x00100801u, 0x28000000u}; - const auto consumed = - HwCtxSetStencilInfo(processor, 0xC0016900u, Pm4::DB_STENCIL_INFO, payload.data(), 1); - const auto& stencil = processor.GetCtx().GetDepthStencilInfo(); - Require("Pm4StencilInfoValueLane", "standalone register value", - consumed == 1 && - stencil.format == Prospero::StencilFormat::k8UInt && - stencil.texture_compatibility == Prospero::TextureCompatibleStencil::kEnable && - !stencil.expclear_enabled && !stencil.htile_stencil_disabled, - "standalone DB_STENCIL_INFO did not decode its sole payload value"); - std::printf("[host] %-32s ok\n", "Pm4StencilInfoValueLane"); +void CheckPm4StencilInfoValueLane(RenderContext &renderer) { + CommandProcessor processor(renderer); + constexpr std::array payload{0x00100801u, 0x28000000u}; + const auto consumed = HwCtxSetStencilInfo( + processor, 0xC0016900u, Pm4::DB_STENCIL_INFO, payload.data(), 1); + const auto &stencil = processor.GetCtx().GetDepthStencilInfo(); + Require("Pm4StencilInfoValueLane", "standalone register value", + consumed == 1 && stencil.format == Prospero::StencilFormat::k8UInt && + stencil.texture_compatibility == + Prospero::TextureCompatibleStencil::kEnable && + !stencil.expclear_enabled && !stencil.htile_stencil_disabled, + "standalone DB_STENCIL_INFO did not decode its sole payload value"); + std::printf("[host] %-32s ok\n", "Pm4StencilInfoValueLane"); } -void CheckPm4PolygonOffsetRegisters(RenderContext& renderer) { - GraphicsInitJmpTables(); - CommandProcessor processor(renderer); - const auto& defaults = processor.GetCtx().GetPolyOffset(); - Require("Pm4PolygonOffset", "format defaults", - defaults.neg_num_db_bits == -23 && defaults.db_is_float_fmt, - "polygon offset Z format did not default to D32F"); +void CheckPm4PolygonOffsetRegisters(RenderContext &renderer) { + GraphicsInitJmpTables(); + CommandProcessor processor(renderer); + const auto &defaults = processor.GetCtx().GetPolyOffset(); + Require("Pm4PolygonOffset", "format defaults", + defaults.neg_num_db_bits == -23 && defaults.db_is_float_fmt, + "polygon offset Z format did not default to D32F"); - constexpr std::array payload { - 0x000000f0u, std::bit_cast(2.5f), std::bit_cast(16.0f), - std::bit_cast(-3.25f), std::bit_cast(32.0f), - std::bit_cast(4.5f)}; - const auto consumed = HwCtxSetPolyOffsetRegisters( - processor, KYTY_PM4(8, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO), - Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, payload.data(), 0); - const auto& direct = processor.GetCtx().GetPolyOffset(); - Require("Pm4PolygonOffset", "direct registers", - consumed == payload.size() && direct.neg_num_db_bits == -16 && - !direct.db_is_float_fmt && direct.clamp == 2.5f && direct.front_scale == 16.0f && - direct.front_offset == -3.25f && direct.back_scale == 32.0f && - direct.back_offset == 4.5f, - "direct polygon offset register packet was decoded incorrectly"); + constexpr std::array payload{0x000000f0u, + std::bit_cast(2.5f), + std::bit_cast(16.0f), + std::bit_cast(-3.25f), + std::bit_cast(32.0f), + std::bit_cast(4.5f)}; + const auto consumed = HwCtxSetPolyOffsetRegisters( + processor, KYTY_PM4(8, Pm4::IT_SET_CONTEXT_REG, Pm4::R_ZERO), + Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, payload.data(), 0); + const auto &direct = processor.GetCtx().GetPolyOffset(); + Require("Pm4PolygonOffset", "direct registers", + consumed == payload.size() && direct.neg_num_db_bits == -16 && + !direct.db_is_float_fmt && direct.clamp == 2.5f && + direct.front_scale == 16.0f && direct.front_offset == -3.25f && + direct.back_scale == 32.0f && direct.back_offset == 4.5f, + "direct polygon offset register packet was decoded incorrectly"); - g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL]( - processor, Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, 0x000001e9u); - g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE]( - processor, Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE, std::bit_cast(24.0f)); - const auto& indirect = processor.GetCtx().GetPolyOffset(); - Require("Pm4PolygonOffset", "indirect registers", - indirect.neg_num_db_bits == -23 && indirect.db_is_float_fmt && - indirect.front_scale == 24.0f && indirect.front_offset == -3.25f, - "indirect polygon offset register write was decoded incorrectly"); - std::printf("[host] %-32s ok\n", "Pm4PolygonOffset"); + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL]( + processor, Pm4::PA_SU_POLY_OFFSET_DB_FMT_CNTL, 0x000001e9u); + g_hw_ctx_indirect_func[Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE]( + processor, Pm4::PA_SU_POLY_OFFSET_FRONT_SCALE, + std::bit_cast(24.0f)); + const auto &indirect = processor.GetCtx().GetPolyOffset(); + Require("Pm4PolygonOffset", "indirect registers", + indirect.neg_num_db_bits == -23 && indirect.db_is_float_fmt && + indirect.front_scale == 24.0f && indirect.front_offset == -3.25f, + "indirect polygon offset register write was decoded incorrectly"); + std::printf("[host] %-32s ok\n", "Pm4PolygonOffset"); } -void CheckPm4ContextStateOperations(RenderContext& renderer) { - GraphicsInitJmpTables(); - CommandProcessor processor(renderer); - constexpr auto primitive_type = static_cast(0x35u); - struct AgcCommandBufferLayout { - using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer*, uint32_t, void*); +void CheckPm4ContextStateOperations(RenderContext &renderer) { + GraphicsInitJmpTables(); + CommandProcessor processor(renderer); + constexpr auto primitive_type = static_cast(0x35u); + struct AgcCommandBufferLayout { + using Callback = KYTY_SYSV_ABI bool (*)(Gen5::CommandBuffer *, uint32_t, + void *); - uint32_t* bottom; - uint32_t* top; - uint32_t* cursor_up; - uint32_t* cursor_down; - Callback callback; - void* user_data; - uint32_t reserved_dw; - }; - static_assert(offsetof(AgcCommandBufferLayout, cursor_up) == 0x10); - static_assert(offsetof(AgcCommandBufferLayout, reserved_dw) == 0x30); + uint32_t *bottom; + uint32_t *top; + uint32_t *cursor_up; + uint32_t *cursor_down; + Callback callback; + void *user_data; + uint32_t reserved_dw; + }; + static_assert(offsetof(AgcCommandBufferLayout, cursor_up) == 0x10); + static_assert(offsetof(AgcCommandBufferLayout, reserved_dw) == 0x30); - HW::RenderControl original_control {}; - original_control.depth_clear_enable = true; - original_control.stencil_clear_enable = true; - processor.GetCtx().SetRenderControl(original_control); - processor.GetCtx().SetRenderTargetMask(0x1234abcd); - processor.GetUcfg().SetPrimitiveType(primitive_type); - g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS](processor, Pm4::SPI_SHADER_PGM_LO_PS, - 0x00123456); - const auto shader_address = processor.GetShCtx().GetPs().ps_regs.data_addr; - const auto invoke = [&](ContextStateOperation operation) { - std::array packet {}; - std::array segment_sizes {}; - AgcCommandBufferLayout dcb {packet.data(), packet.data() + packet.size(), packet.data(), - packet.data() + packet.size(), nullptr, nullptr, 0}; - const auto operation_value = static_cast(operation); - size_t segment_count = 0; - switch (operation) { - case ContextStateOperation::Clear: segment_sizes = {5}; segment_count = 1; break; - case ContextStateOperation::Push: - segment_sizes = {5, 8, 9, 3, 2}; - segment_count = 5; - break; - case ContextStateOperation::Pop: - segment_sizes = {3, 5, 8, 9, 2}; - segment_count = 5; - break; - case ContextStateOperation::PushClear: - segment_sizes = {5, 8, 9, 3, 2, 5}; - segment_count = 6; - break; - } - const auto size_dw = - static_cast(Gen5::GraphicsDcbContextStateOpGetSize(operation_value) / 4); - auto* emitted = Gen5::GraphicsDcbContextStateOp( - reinterpret_cast(&dcb), operation_value); - bool packet_matches = true; - size_t segment_offset = 0; - for (size_t i = 0; i < segment_count; i++) { - const auto segment_size = segment_sizes[i]; - const auto selector = (i == 0 ? Pm4::R_CONTEXT_STATE : Pm4::R_ZERO); - packet_matches &= - packet[segment_offset] == KYTY_PM4(segment_size, Pm4::IT_NOP, selector); - for (size_t j = 1; j < segment_size; j++) { - packet_matches &= packet[segment_offset + j] == - (i == 0 && j == 1 ? operation_value : 0u); - } - segment_offset += segment_size; - } - Require("Pm4ContextState", "HLE packet", - emitted == packet.data() && dcb.cursor_up == packet.data() + size_dw && - segment_offset == size_dw && packet_matches, - "context-state HLE packet does not match its real allocation size"); - Pm4Execution execution; - return processor.Process(execution, packet.data(), size_dw); - }; + HW::RenderControl original_control{}; + original_control.depth_clear_enable = true; + original_control.stencil_clear_enable = true; + processor.GetCtx().SetRenderControl(original_control); + processor.GetCtx().SetRenderTargetMask(0x1234abcd); + processor.GetUcfg().SetPrimitiveType(primitive_type); + g_hw_sh_indirect_func[Pm4::SPI_SHADER_PGM_LO_PS]( + processor, Pm4::SPI_SHADER_PGM_LO_PS, 0x00123456); + const auto shader_address = processor.GetShCtx().GetPs().ps_regs.data_addr; + const auto invoke = [&](ContextStateOperation operation) { + std::array packet{}; + std::array segment_sizes{}; + AgcCommandBufferLayout dcb{packet.data(), + packet.data() + packet.size(), + packet.data(), + packet.data() + packet.size(), + nullptr, + nullptr, + 0}; + const auto operation_value = static_cast(operation); + size_t segment_count = 0; + switch (operation) { + case ContextStateOperation::Clear: + segment_sizes = {5}; + segment_count = 1; + break; + case ContextStateOperation::Push: + segment_sizes = {5, 8, 9, 3, 2}; + segment_count = 5; + break; + case ContextStateOperation::Pop: + segment_sizes = {3, 5, 8, 9, 2}; + segment_count = 5; + break; + case ContextStateOperation::PushClear: + segment_sizes = {5, 8, 9, 3, 2, 5}; + segment_count = 6; + break; + } + const auto size_dw = static_cast( + Gen5::GraphicsDcbContextStateOpGetSize(operation_value) / 4); + auto *emitted = Gen5::GraphicsDcbContextStateOp( + reinterpret_cast(&dcb), operation_value); + bool packet_matches = true; + size_t segment_offset = 0; + for (size_t i = 0; i < segment_count; i++) { + const auto segment_size = segment_sizes[i]; + const auto selector = (i == 0 ? Pm4::R_CONTEXT_STATE : Pm4::R_ZERO); + packet_matches &= packet[segment_offset] == + KYTY_PM4(segment_size, Pm4::IT_NOP, selector); + for (size_t j = 1; j < segment_size; j++) { + packet_matches &= packet[segment_offset + j] == + (i == 0 && j == 1 ? operation_value : 0u); + } + segment_offset += segment_size; + } + Require("Pm4ContextState", "HLE packet", + emitted == packet.data() && + dcb.cursor_up == packet.data() + size_dw && + segment_offset == size_dw && packet_matches, + "context-state HLE packet does not match its real allocation size"); + Pm4Execution execution; + return processor.Process(execution, packet.data(), size_dw); + }; - Require("Pm4ContextState", "push-clear", - invoke(ContextStateOperation::PushClear) == Pm4ProcessResult::Complete && - !processor.GetCtx().GetRenderControl().depth_clear_enable && - !processor.GetCtx().GetRenderControl().stencil_clear_enable && - processor.GetCtx().GetRenderTargetMask() == HW::Context {}.GetRenderTargetMask() && - processor.GetUcfg().GetPrimType() == primitive_type && - processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address, - "push-clear did not save and clear only Cx state"); + Require("Pm4ContextState", "push-clear", + invoke(ContextStateOperation::PushClear) == + Pm4ProcessResult::Complete && + !processor.GetCtx().GetRenderControl().depth_clear_enable && + !processor.GetCtx().GetRenderControl().stencil_clear_enable && + processor.GetCtx().GetRenderTargetMask() == + HW::Context{}.GetRenderTargetMask() && + processor.GetUcfg().GetPrimType() == primitive_type && + processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address, + "push-clear did not save and clear only Cx state"); - processor.GetCtx().SetRenderTargetMask(0x55aa55aa); - Require("Pm4ContextState", "pop", - invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete && - processor.GetCtx().GetRenderControl().depth_clear_enable && - processor.GetCtx().GetRenderControl().stencil_clear_enable && - processor.GetCtx().GetRenderTargetMask() == 0x1234abcd, - "pop did not restore the complete saved Cx state"); + processor.GetCtx().SetRenderTargetMask(0x55aa55aa); + Require("Pm4ContextState", "pop", + invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete && + processor.GetCtx().GetRenderControl().depth_clear_enable && + processor.GetCtx().GetRenderControl().stencil_clear_enable && + processor.GetCtx().GetRenderTargetMask() == 0x1234abcd, + "pop did not restore the complete saved Cx state"); - Require("Pm4ContextState", "push", - invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete && - processor.GetCtx().GetRenderTargetMask() == 0x1234abcd, - "push changed live Cx state"); - processor.GetCtx().SetRenderTargetMask(0x01020304); - Require("Pm4ContextState", "push-pop restore", - invoke(ContextStateOperation::Pop) == - Pm4ProcessResult::Complete && - processor.GetCtx().GetRenderTargetMask() == 0x1234abcd, - "ordinary push/pop did not restore Cx state"); + Require("Pm4ContextState", "push", + invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete && + processor.GetCtx().GetRenderTargetMask() == 0x1234abcd, + "push changed live Cx state"); + processor.GetCtx().SetRenderTargetMask(0x01020304); + Require("Pm4ContextState", "push-pop restore", + invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete && + processor.GetCtx().GetRenderTargetMask() == 0x1234abcd, + "ordinary push/pop did not restore Cx state"); - Require("Pm4ContextState", "clear", - invoke(ContextStateOperation::Clear) == - Pm4ProcessResult::Complete && - !processor.GetCtx().GetRenderControl().depth_clear_enable && - processor.GetCtx().GetRenderTargetMask() == - HW::Context{}.GetRenderTargetMask() && - processor.GetUcfg().GetPrimType() == primitive_type && - processor.GetShCtx().GetPs().ps_regs.data_addr == - shader_address, - "clear reset state outside the Cx register domain"); + Require("Pm4ContextState", "clear", + invoke(ContextStateOperation::Clear) == Pm4ProcessResult::Complete && + !processor.GetCtx().GetRenderControl().depth_clear_enable && + processor.GetCtx().GetRenderTargetMask() == + HW::Context{}.GetRenderTargetMask() && + processor.GetUcfg().GetPrimType() == primitive_type && + processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address, + "clear reset state outside the Cx register domain"); - processor.GetCtx().SetRenderTargetMask(0x0badc0de); - Require("Pm4ContextState", "push before clear-state packet", - invoke(ContextStateOperation::Push) == - Pm4ProcessResult::Complete, - "push before CLEAR_STATE failed"); - processor.GetCtx().SetRenderTargetMask(0xfeedface); - std::array clear_state_packet{0xc0001200, 0}; - Pm4Execution clear_state_execution; - Require("Pm4ContextState", "clear-state packet", - processor.Process( - clear_state_execution, clear_state_packet.data(), - clear_state_packet.size()) == Pm4ProcessResult::Complete && - processor.GetCtx().GetRenderTargetMask() == - HW::Context{}.GetRenderTargetMask(), - "CLEAR_STATE did not clear the current Cx state"); - Require("Pm4ContextState", "pop after clear-state packet", - invoke(ContextStateOperation::Pop) == - Pm4ProcessResult::Complete && - processor.GetCtx().GetRenderTargetMask() == 0x0badc0de, - "CLEAR_STATE discarded the pushed Cx state"); + processor.GetCtx().SetRenderTargetMask(0x0badc0de); + Require("Pm4ContextState", "push before clear-state packet", + invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete, + "push before CLEAR_STATE failed"); + processor.GetCtx().SetRenderTargetMask(0xfeedface); + std::array clear_state_packet{0xc0001200, 0}; + Pm4Execution clear_state_execution; + Require("Pm4ContextState", "clear-state packet", + processor.Process(clear_state_execution, clear_state_packet.data(), + clear_state_packet.size()) == + Pm4ProcessResult::Complete && + processor.GetCtx().GetRenderTargetMask() == + HW::Context{}.GetRenderTargetMask(), + "CLEAR_STATE did not clear the current Cx state"); + Require("Pm4ContextState", "pop after clear-state packet", + invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete && + processor.GetCtx().GetRenderTargetMask() == 0x0badc0de, + "CLEAR_STATE discarded the pushed Cx state"); - processor.GetCtx().SetRenderTargetMask(0xabcdef01); - Require("Pm4ContextState", "push before processor reset", - invoke(ContextStateOperation::Push) == - Pm4ProcessResult::Complete, - "push before reset failed"); - processor.Reset(); - processor.GetCtx().SetRenderTargetMask(0x76543210); - Require("Pm4ContextState", "processor reset discards push", - invoke(ContextStateOperation::Push) == - Pm4ProcessResult::Complete && - invoke(ContextStateOperation::Pop) == - Pm4ProcessResult::Complete && - processor.GetCtx().GetRenderTargetMask() == 0x76543210, - "processor reset retained an invalid saved Cx state"); + processor.GetCtx().SetRenderTargetMask(0xabcdef01); + Require("Pm4ContextState", "push before processor reset", + invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete, + "push before reset failed"); + processor.Reset(); + processor.GetCtx().SetRenderTargetMask(0x76543210); + Require("Pm4ContextState", "processor reset discards push", + invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete && + invoke(ContextStateOperation::Pop) == + Pm4ProcessResult::Complete && + processor.GetCtx().GetRenderTargetMask() == 0x76543210, + "processor reset retained an invalid saved Cx state"); - Require("Pm4ContextState", "HLE packet size", - Gen5::GraphicsDcbContextStateOpGetSize(0) == 20 && - Gen5::GraphicsDcbContextStateOpGetSize(1) == 108 && - Gen5::GraphicsDcbContextStateOpGetSize(2) == 108 && - Gen5::GraphicsDcbContextStateOpGetSize(3) == 128 && - Gen5::GraphicsDcbContextStateOpGetSize(4) == 0, - "context-state HLE sizes do not match libSceAgc"); - std::printf("[host] %-32s ok\n", "Pm4ContextState"); + Require("Pm4ContextState", "HLE packet size", + Gen5::GraphicsDcbContextStateOpGetSize(0) == 20 && + Gen5::GraphicsDcbContextStateOpGetSize(1) == 108 && + Gen5::GraphicsDcbContextStateOpGetSize(2) == 108 && + Gen5::GraphicsDcbContextStateOpGetSize(3) == 128 && + Gen5::GraphicsDcbContextStateOpGetSize(4) == 0, + "context-state HLE sizes do not match libSceAgc"); + std::printf("[host] %-32s ok\n", "Pm4ContextState"); } -void CheckPm4WaitResume(RenderContext& renderer) { - GraphicsInitJmpTables(); - CommandProcessor processor(renderer); +void CheckPm4WaitResume(RenderContext &renderer) { + GraphicsInitJmpTables(); + CommandProcessor processor(renderer); - uint32_t label = 0; - uint32_t prefix = 0; - uint32_t child_observation = UINT32_MAX; - uint32_t suffix = 0; - const auto address = [](const void* value) { return reinterpret_cast(value); }; + uint32_t label = 0; + uint32_t prefix = 0; + uint32_t child_observation = UINT32_MAX; + uint32_t suffix = 0; + const auto address = [](const void *value) { + return reinterpret_cast(value); + }; - std::array child {}; - child[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - child[1] = 0; - child[2] = static_cast(address(&child_observation)); - child[3] = static_cast(address(&child_observation) >> 32u); - child[4] = 0; - child[5] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); - child[6] = static_cast(address(&label)); - child[7] = static_cast(address(&label) >> 32u); - child[8] = UINT32_MAX; - child[9] = 1; - child[10] = 0x10u | 3u; + std::array child{}; + child[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + child[1] = 0; + child[2] = static_cast(address(&child_observation)); + child[3] = static_cast(address(&child_observation) >> 32u); + child[4] = 0; + child[5] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_MEM_32); + child[6] = static_cast(address(&label)); + child[7] = static_cast(address(&label) >> 32u); + child[8] = UINT32_MAX; + child[9] = 1; + child[10] = 0x10u | 3u; - std::array nested {}; - nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); - nested[1] = static_cast(address(child.data())); - nested[2] = static_cast(address(child.data()) >> 32u); - nested[3] = 0x0f200000u | static_cast(child.size()); + std::array nested{}; + nested[0] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + nested[1] = static_cast(address(child.data())); + nested[2] = static_cast(address(child.data()) >> 32u); + nested[3] = 0x0f200000u | static_cast(child.size()); - std::array commands {}; - commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - commands[1] = 0; - commands[2] = static_cast(address(&prefix)); - commands[3] = static_cast(address(&prefix) >> 32u); - commands[4] = 11; - commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); - commands[6] = static_cast(address(nested.data())); - commands[7] = static_cast(address(nested.data()) >> 32u); - commands[8] = 0x0f200000u | static_cast(nested.size()); - commands[9] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - commands[10] = 0; - commands[11] = static_cast(address(&suffix)); - commands[12] = static_cast(address(&suffix) >> 32u); - commands[13] = 22; + std::array commands{}; + commands[0] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[1] = 0; + commands[2] = static_cast(address(&prefix)); + commands[3] = static_cast(address(&prefix) >> 32u); + commands[4] = 11; + commands[5] = KYTY_PM4(4, Pm4::IT_INDIRECT_BUFFER, 0); + commands[6] = static_cast(address(nested.data())); + commands[7] = static_cast(address(nested.data()) >> 32u); + commands[8] = 0x0f200000u | static_cast(nested.size()); + commands[9] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[10] = 0; + commands[11] = static_cast(address(&suffix)); + commands[12] = static_cast(address(&suffix) >> 32u); + commands[13] = 22; - Pm4Execution execution; - Require("Pm4WaitResume", "suspend", - processor.Process(execution, commands.data(), commands.size()) == - Pm4ProcessResult::Blocked && - prefix == 11 && child_observation == 0 && suffix == 0, - "blocked indirect wait did not preserve its command position"); + Pm4Execution execution; + Require("Pm4WaitResume", "suspend", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Blocked && + prefix == 11 && child_observation == 0 && suffix == 0, + "blocked indirect wait did not preserve its command position"); - label = 1; - child[4] = 1; - Require("Pm4WaitResume", "resume", - processor.Process(execution, commands.data(), commands.size()) == - Pm4ProcessResult::Complete && - prefix == 11 && child_observation == 0 && suffix == 22, - "resumed indirect wait replayed a child or did not finish its parent"); - std::printf("[host] %-32s ok\n", "Pm4WaitResume"); + label = 1; + child[4] = 1; + Require( + "Pm4WaitResume", "resume", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Complete && + prefix == 11 && child_observation == 0 && suffix == 22, + "resumed indirect wait replayed a child or did not finish its parent"); + std::printf("[host] %-32s ok\n", "Pm4WaitResume"); } -void CheckPm4CeCompletion(RenderContext& renderer) { - GraphicsInitJmpTables(); - CommandProcessor processor(renderer); - uint32_t suffix = 0; - const auto address = reinterpret_cast(&suffix); +void CheckPm4CeCompletion(RenderContext &renderer) { + GraphicsInitJmpTables(); + CommandProcessor processor(renderer); + uint32_t suffix = 0; + const auto address = reinterpret_cast(&suffix); - std::array commands {}; - commands[0] = 0xc0008600u; - commands[1] = 1; - commands[2] = 0xc0008500u; - commands[3] = 0; - commands[4] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); - commands[5] = 0; - commands[6] = static_cast(address); - commands[7] = static_cast(address >> 32u); - commands[8] = 33; + std::array commands{}; + commands[0] = 0xc0008600u; + commands[1] = 1; + commands[2] = 0xc0008500u; + commands[3] = 0; + commands[4] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0); + commands[5] = 0; + commands[6] = static_cast(address); + commands[7] = static_cast(address >> 32u); + commands[8] = 33; - processor.ResetDeCe(); - Pm4Execution execution; - Require("Pm4CeCompletion", "wait", - processor.Process(execution, commands.data(), commands.size()) == - Pm4ProcessResult::Blocked && - suffix == 0, - "DE did not wait for an active CE stream"); + processor.ResetDeCe(); + Pm4Execution execution; + Require("Pm4CeCompletion", "wait", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Blocked && + suffix == 0, + "DE did not wait for an active CE stream"); - std::array ce_increment {0xc0008400u, 1}; - Pm4Execution ce_execution; - Require("Pm4CeCompletion", "CE increment", - processor.Process(ce_execution, ce_increment.data(), ce_increment.size()) == - Pm4ProcessResult::Complete, - "CE counter increment did not complete"); - Require("Pm4CeCompletion", "counter gate", - processor.Process(execution, commands.data(), commands.size()) == - Pm4ProcessResult::Complete && - suffix == 33, - "DE did not resume after CE advanced or failed to increment its counter"); + std::array ce_increment{0xc0008400u, 1}; + Pm4Execution ce_execution; + Require("Pm4CeCompletion", "CE increment", + processor.Process(ce_execution, ce_increment.data(), + ce_increment.size()) == Pm4ProcessResult::Complete, + "CE counter increment did not complete"); + Require( + "Pm4CeCompletion", "counter gate", + processor.Process(execution, commands.data(), commands.size()) == + Pm4ProcessResult::Complete && + suffix == 33, + "DE did not resume after CE advanced or failed to increment its counter"); - suffix = 0; - Pm4Execution balanced_execution; - Require("Pm4CeCompletion", "balanced wait", - processor.Process(balanced_execution, commands.data(), commands.size()) == - Pm4ProcessResult::Blocked && - suffix == 0, - "balanced CE/DE counters did not gate the next DE packet"); - processor.SetCeComplete(true); - Require("Pm4CeCompletion", "stream complete", - processor.Process(balanced_execution, commands.data(), commands.size()) == - Pm4ProcessResult::Complete && - suffix == 33, - "DE remained blocked after the CE stream completed"); - std::printf("[host] %-32s ok\n", "Pm4CeCompletion"); + suffix = 0; + Pm4Execution balanced_execution; + Require("Pm4CeCompletion", "balanced wait", + processor.Process(balanced_execution, commands.data(), + commands.size()) == Pm4ProcessResult::Blocked && + suffix == 0, + "balanced CE/DE counters did not gate the next DE packet"); + processor.SetCeComplete(true); + Require("Pm4CeCompletion", "stream complete", + processor.Process(balanced_execution, commands.data(), + commands.size()) == Pm4ProcessResult::Complete && + suffix == 33, + "DE remained blocked after the CE stream completed"); + std::printf("[host] %-32s ok\n", "Pm4CeCompletion"); } } // namespace } // namespace Libs::Graphics -int main(int argc, char** argv) { - using namespace Libs::Graphics; +int main(int argc, char **argv) { + using namespace Libs::Graphics; - std::setvbuf(stdout, nullptr, _IONBF, 0); - EnsureConfigInitialized(); - CheckLeastRecentlyUsedCacheOrdering(); - if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) { - CheckClipControlDepthClipState(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--reference-clock-only") == 0) { - CheckReferenceClockScale(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--scheduler-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckSchedulerTimeline(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckGpuMappedRangeLifecycle(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckStreamBufferRing(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckGpuTilerCpuParity(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckGpuCommandLane(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) { - VulkanHarness vulkan; - CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer()); - CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) { - CheckDepthAttachmentWrites(); - CheckDynamicRenderingState(); - VulkanHarness vulkan; - vulkan.CheckRenderExecutorColorVolumeDiscovery(); - vulkan.CheckRenderExecutorStencilBindingDiscovery(); - vulkan.CheckUnifiedTextureCacheFlow(); - vulkan.CheckBgra16Readback(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckUnifiedTextureCacheFlow(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckUnifiedImageViewCache(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--image-view-cache-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckUnifiedImageViewCache(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckUnifiedImageViewCache(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) { - CheckDepthTargetFootprints(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckUnifiedTextureCacheFlow(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-gc-only") == 0) { - VulkanHarness vulkan; - vulkan.CheckBufferCacheDirtyGarbageCollection(); - return 0; - } + std::setvbuf(stdout, nullptr, _IONBF, 0); + EnsureConfigInitialized(); + CheckLeastRecentlyUsedCacheOrdering(); + if (argc == 2 && std::strcmp(argv[1], "--clip-control-only") == 0) { + CheckClipControlDepthClipState(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--reference-clock-only") == 0) { + CheckReferenceClockScale(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--scheduler-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckSchedulerTimeline(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--mapped-range-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGpuMappedRangeLifecycle(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--stream-buffer-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckStreamBufferRing(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--gpu-tiler-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGpuTilerCpuParity(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckGpuCommandLane(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--context-state-only") == 0) { + VulkanHarness vulkan; + CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer()); + CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--image-overlap-only") == 0) { + CheckDepthAttachmentWrites(); + CheckDynamicRenderingState(); + VulkanHarness vulkan; + vulkan.CheckRenderExecutorColorVolumeDiscovery(); + vulkan.CheckRenderExecutorStencilBindingDiscovery(); + vulkan.CheckUnifiedTextureCacheFlow(); + vulkan.CheckBgra16Readback(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--htile-clear-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckUnifiedTextureCacheFlow(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--layered-image-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckUnifiedImageViewCache(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--image-view-cache-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckUnifiedImageViewCache(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--storage-sampled-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckUnifiedImageViewCache(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--depth-readback-only") == 0) { + CheckDepthTargetFootprints(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-range-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckUnifiedTextureCacheFlow(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--buffer-cache-gc-only") == 0) { + VulkanHarness vulkan; + vulkan.CheckBufferCacheDirtyGarbageCollection(); + return 0; + } #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) { - RunReverseRenderTargetDeathCase(); - } - if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-only") == 0) { - CheckRenderTargetFormatContract(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--standard64-rt-only") == 0) { - CheckStandard64RenderTargetTileRoundTrip(); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) { - VulkanHarness vulkan; - CheckSampledColorViews(); - CheckSampledVideoOutView(vulkan.RuntimeRenderer()); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--image-transition-only") == 0) { - VulkanHarness vulkan; - CheckImageTransitionState(vulkan.RuntimeRenderer()); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) { - VulkanHarness vulkan; - CheckSampledDepthResource(); - CheckSampledDepthDescriptor(vulkan.RuntimeRenderer()); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) { - CheckSampledColorViews(); - CheckBasicStorageTextureDescriptor(); - VulkanHarness vulkan; - vulkan.CheckUnifiedImageViewCache(); - RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle()); - return 0; - } - if (argc == 2 && std::strcmp(argv[1], "--storage-yzwx-only") == 0) { - CheckSampledColorViews(); - CheckBasicStorageTextureDescriptor(); - CheckStorageTextureGpuOwnedRebindState(); - VulkanHarness vulkan; - RunCase(&vulkan, ImageStoreYzwxUsesInverseSwizzle()); - return 0; - } - if (argc == 3 && std::strcmp(argv[1], "--image-view-death") == 0) { - RunImageViewDeathCase(argv[2]); - } - if (argc == 3 && std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) { - RunStorageTextureDescriptorDeathCase(argv[2]); - } - if (argc != 1) { - std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); - return 2; - } - VulkanHarness vulkan; - CheckRenderTargetFormatContract(); - CheckSampledColorViews(); - CheckSampledVideoOutView(vulkan.RuntimeRenderer()); - CheckImageTransitionState(vulkan.RuntimeRenderer()); - CheckSampledDepthResource(); - CheckSampledDepthDescriptor(vulkan.RuntimeRenderer()); - CheckBasicStorageTextureDescriptor(); - CheckStorageTextureLinearUploadLayout(); - CheckStorageTextureDepthTileUploadLayout(); - CheckStorageImageSwizzleSpecializationId(); - CheckStandard64RenderTargetTileRoundTrip(); - CheckStorageTextureVolumeUploadLayout(); - CheckStorageTextureVolumeMipRegions(); - CheckStorageTextureGpuOwnedRebindState(); - CheckNativeMsaaState(); - CheckPs5DepthRegisterDecoding(); - CheckDepthHtileStencilCompatibility(); - CheckStencilAttachmentAccess(); - CheckDepthAttachmentWrites(); - CheckDynamicRenderingState(); - CheckDepthTargetFootprints(); - CheckSharedFenceResourceLifetime(); + if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-death") == 0) { + RunReverseRenderTargetDeathCase(); + } + if (argc == 2 && std::strcmp(argv[1], "--reverse-rt-only") == 0) { + CheckRenderTargetFormatContract(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--standard64-rt-only") == 0) { + CheckStandard64RenderTargetTileRoundTrip(); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--image-view-only") == 0) { + VulkanHarness vulkan; + CheckSampledColorViews(); + CheckSampledVideoOutView(vulkan.RuntimeRenderer()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--image-transition-only") == 0) { + VulkanHarness vulkan; + CheckImageTransitionState(vulkan.RuntimeRenderer()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--sampled-depth-resource-only") == 0) { + VulkanHarness vulkan; + CheckSampledDepthResource(); + CheckSampledDepthDescriptor(vulkan.RuntimeRenderer()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--storage-bgra-only") == 0) { + CheckSampledColorViews(); + CheckBasicStorageTextureDescriptor(); + VulkanHarness vulkan; + vulkan.CheckUnifiedImageViewCache(); + RunCase(&vulkan, ImageStoreBgraUsesInverseSwizzle()); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--storage-yzwx-only") == 0) { + CheckSampledColorViews(); + CheckBasicStorageTextureDescriptor(); + CheckStorageTextureGpuOwnedRebindState(); + VulkanHarness vulkan; + RunCase(&vulkan, ImageStoreYzwxUsesInverseSwizzle()); + return 0; + } + if (argc == 3 && std::strcmp(argv[1], "--image-view-death") == 0) { + RunImageViewDeathCase(argv[2]); + } + if (argc == 3 && + std::strcmp(argv[1], "--storage-texture-descriptor-death") == 0) { + RunStorageTextureDescriptorDeathCase(argv[2]); + } + if (argc != 1) { + std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); + return 2; + } + VulkanHarness vulkan; + CheckRenderTargetFormatContract(); + CheckSampledColorViews(); + CheckSampledVideoOutView(vulkan.RuntimeRenderer()); + CheckImageTransitionState(vulkan.RuntimeRenderer()); + CheckSampledDepthResource(); + CheckSampledDepthDescriptor(vulkan.RuntimeRenderer()); + CheckBasicStorageTextureDescriptor(); + CheckStorageTextureLinearUploadLayout(); + CheckStorageTextureDepthTileUploadLayout(); + CheckStorageImageSwizzleSpecializationId(); + CheckStandard64RenderTargetTileRoundTrip(); + CheckStorageTextureVolumeUploadLayout(); + CheckStorageTextureVolumeMipRegions(); + CheckStorageTextureGpuOwnedRebindState(); + CheckNativeMsaaState(); + CheckPs5DepthRegisterDecoding(); + CheckDepthHtileStencilCompatibility(); + CheckStencilAttachmentAccess(); + CheckDepthAttachmentWrites(); + CheckDynamicRenderingState(); + CheckDepthTargetFootprints(); + CheckSharedFenceResourceLifetime(); #else - if (argc != 1) { - std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); - return 2; - } - VulkanHarness vulkan; + if (argc != 1) { + std::fprintf(stderr, "unknown test selector: %s\n", argv[1]); + return 2; + } + VulkanHarness vulkan; #endif - CheckClipControlDepthClipState(); - CheckReferenceClockScale(); - CheckVulkan13FeatureRequirements(); - CheckPm4AcquireMemNoOp(vulkan.RuntimeRenderer()); - CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer()); - CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer()); - CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); - CheckPm4WaitResume(vulkan.RuntimeRenderer()); - CheckPm4CeCompletion(vulkan.RuntimeRenderer()); - CheckEmbeddedFetchVertexOffset(); - CheckEmbeddedFetchLaneSpill(); - CheckRectListShaders(); - CheckPs5GameExampleImageClearRuntimeShape(); - vulkan.CheckSchedulerTimeline(); - vulkan.CheckGpuMappedRangeLifecycle(); - vulkan.CheckStreamBufferRing(); - vulkan.CheckCommandPoolGrowth(); - vulkan.CheckGpuTilerCpuParity(); + CheckClipControlDepthClipState(); + CheckReferenceClockScale(); + CheckVulkan13FeatureRequirements(); + CheckPm4AcquireMemNoOp(vulkan.RuntimeRenderer()); + CheckPm4StencilInfoValueLane(vulkan.RuntimeRenderer()); + CheckPm4PolygonOffsetRegisters(vulkan.RuntimeRenderer()); + CheckPm4ContextStateOperations(vulkan.RuntimeRenderer()); + CheckPm4WaitResume(vulkan.RuntimeRenderer()); + CheckPm4CeCompletion(vulkan.RuntimeRenderer()); + CheckEmbeddedFetchVertexOffset(); + CheckEmbeddedFetchLaneSpill(); + CheckRectListShaders(); + CheckPs5GameExampleImageClearRuntimeShape(); + vulkan.CheckSchedulerTimeline(); + vulkan.CheckGpuMappedRangeLifecycle(); + vulkan.CheckStreamBufferRing(); + vulkan.CheckCommandPoolGrowth(); + vulkan.CheckGpuTilerCpuParity(); #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS - vulkan.CheckRenderExecutorColorVolumeDiscovery(); - vulkan.CheckRenderExecutorStencilBindingDiscovery(); - vulkan.CheckUnifiedTextureCacheFlow(); - vulkan.CheckBgra16Readback(); - vulkan.CheckBufferCacheDirtyGarbageCollection(); + vulkan.CheckRenderExecutorColorVolumeDiscovery(); + vulkan.CheckRenderExecutorStencilBindingDiscovery(); + vulkan.CheckUnifiedTextureCacheFlow(); + vulkan.CheckBgra16Readback(); + vulkan.CheckBufferCacheDirtyGarbageCollection(); #endif - vulkan.CheckUnifiedImageViewCache(); - const auto tests = MakeCases(); - const auto graphics_tests = MakeGraphicsCases(); - CheckOpcodeCoverage(tests, graphics_tests); - for (const auto& test: tests) { - RunCase(&vulkan, test); - } - for (const auto& test: MakeSkippedCases()) { - std::printf("[skip] %-32s %s\n", test.name, test.reason); - } - for (const auto& test: graphics_tests) { - RunGraphicsCase(&vulkan, test); - } - vulkan.CheckGpuCommandLane(); - std::printf("ShaderRecompilerComputeTests: all cases passed\n"); - return 0; + vulkan.CheckUnifiedImageViewCache(); + const auto tests = MakeCases(); + const auto graphics_tests = MakeGraphicsCases(); + CheckOpcodeCoverage(tests, graphics_tests); + for (const auto &test : tests) { + RunCase(&vulkan, test); + } + for (const auto &test : MakeSkippedCases()) { + std::printf("[skip] %-32s %s\n", test.name, test.reason); + } + for (const auto &test : graphics_tests) { + RunGraphicsCase(&vulkan, test); + } + vulkan.CheckGpuCommandLane(); + std::printf("ShaderRecompilerComputeTests: all cases passed\n"); + return 0; } diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp index 7d347cbbd..633282e7c 100644 --- a/tests/ShaderStageRuntimeTests.cpp +++ b/tests/ShaderStageRuntimeTests.cpp @@ -1,5 +1,5 @@ -#include "graphics/shader/recompiler/ir/ResourceMaterialization.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/shader.h" #include @@ -8,81 +8,128 @@ namespace { -void Check(bool value, const char* text) { - if (!value) { - std::fprintf(stderr, "ShaderStageRuntimeTests: failed: %s\n", text); - std::abort(); - } +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "ShaderStageRuntimeTests: failed: %s\n", text); + std::abort(); + } } -std::shared_ptr SrtProgram(uint64_t address) { - using namespace Libs::Graphics::ShaderRecompiler::IR; - Program program; - program.stage = Libs::Graphics::ShaderType::Compute; - program.srt_plan_complete = true; - program.srt_patching_complete = true; - program.resource_tracking_complete = true; - program.provenance.values.resize(6); - program.provenance.values[0].op = ScalarValueOp::Undefined; - program.provenance.values[1].op = ScalarValueOp::Unknown; - program.provenance.values[2].op = ScalarValueOp::Constant; - program.provenance.values[2].imm = static_cast(address); - program.provenance.values[3].op = ScalarValueOp::Constant; - program.provenance.values[3].imm = static_cast(address >> 32u); - program.provenance.values[4].op = ScalarValueOp::Constant; - program.provenance.values[5].op = ScalarValueOp::ReadConst; - program.provenance.values[5].args = {2, 3, 4}; - program.srt.reads.push_back({5, 0, 0x40}); - return std::make_shared(std::move(program)); +Libs::Graphics::ShaderRecompiler::IR::Block & +AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + program.values = std::make_shared(); + auto block = std::make_unique(); + auto *result = block.get(); + program.values->blocks.push_back(result); + program.values->block_info.push_back({.id = 0}); + program.values->block_storage.push_back(std::move(block)); + return *result; } -std::shared_ptr UnbasedFlatProgram() { - using namespace Libs::Graphics::ShaderRecompiler::IR; - Program program; - program.stage = Libs::Graphics::ShaderType::Compute; - program.srt_plan_complete = true; - program.srt_patching_complete = true; - program.resource_tracking_complete = true; - AddressResource address; - address.source = ScalarProvenance::Unknown; - address.first_use_pc = 0x80; - address.kind = ResourceKind::Flat; - program.info.addresses.push_back(address); - return std::make_shared(std::move(program)); +std::shared_ptr +SrtProgram(uint64_t address) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + auto &value_block = AddValueBlock(program); + + MemoryInfo memory; + memory.kind = ResourceKind::ScalarBuffer; + memory.planning_only = true; + program.values->memory_info.push_back(memory); + const auto low = Value(static_cast(address)); + const auto high = Value(static_cast(address >> 32u)); + auto &handle = + value_block.AppendNewInst(ValueOpcode::GetAddressResource, {low, high}); + auto &raw = value_block.AppendNewInst( + ValueOpcode::LoadAddressU32, + {Value(&handle), Value(0u), Value(0u), Value(true)}); + raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40}); + program.values->srt_reads.push_back({Value(&raw), 0, 0x40}); + + auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource); + auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst, + {Value(&srt), Value(0u)}); + DescriptorSource source; + source.dwords[0] = Value(&flat); + source.dwords[1] = Value(0u); + source.dword_count = 2; + program.values->descriptor_sources.push_back(source); + return std::make_shared(std::move(program)); +} + +std::shared_ptr +UnbasedFlatProgram() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + AddValueBlock(program); + AddressResource address; + address.source = UINT32_MAX; + address.first_use_pc = 0x80; + address.kind = ResourceKind::Flat; + address.unbased = true; + program.info.addresses.push_back(address); + return std::make_shared(std::move(program)); } void TestMappedSrtUsesDirectReaderByDefault() { - using namespace Libs::Graphics; - const uint32_t dword = 0x12345678; - auto cached_program = SrtProgram(reinterpret_cast(&dword)); - ShaderStageRuntime stage; - std::string error; - Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error), error.c_str()); - Check(stage.program == cached_program && stage.resources != nullptr, - "cache rematerialization did not publish the mapped stage"); - Check(stage.resources->flattened_srt.size() == 1 && stage.resources->flattened_srt[0] == dword, - "cache rematerialization did not use the direct reader by default"); + using namespace Libs::Graphics; + const uint32_t dword = 0x12345678; + auto cached_program = SrtProgram(reinterpret_cast(&dword)); + ShaderStageRuntime stage; + std::string error; + Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error), + error.c_str()); + Check(stage.program == cached_program && stage.resources != nullptr, + "cache rematerialization did not publish the mapped stage"); + Check(stage.resources->flattened_srt.size() == 1 && + stage.resources->flattened_srt[0] == dword, + "cache rematerialization did not use the direct reader by default"); } void TestUnbasedFlatCacheHitFailsClosed() { - using namespace Libs::Graphics; - auto cached_program = UnbasedFlatProgram(); - auto prior_program = std::make_shared(); - auto prior_resources = std::make_shared(); - ShaderStageRuntime stage {prior_program, prior_resources}; - std::string error; - Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error) && - error.find("requires runtime guest-address translation") != std::string::npos, - "unbased FLAT cache hit did not fail without a runtime translator"); - Check(stage.program == prior_program && stage.resources == prior_resources, - "unbased FLAT cache rejection replaced the prior stage snapshot"); + using namespace Libs::Graphics; + auto cached_program = UnbasedFlatProgram(); + auto prior_program = std::make_shared(); + auto prior_resources = + std::make_shared(); + ShaderStageRuntime stage{prior_program, prior_resources}; + std::string error; + Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, &error) && + error.find("requires runtime guest-address translation") != + std::string::npos, + "unbased FLAT cache hit did not fail without a runtime translator"); + Check(stage.program == prior_program && stage.resources == prior_resources, + "unbased FLAT cache rejection replaced the prior stage snapshot"); } } // namespace +namespace Common { + +int DbgExitIfHandler(const char *, const char *, int) { return 1; } + +void DbgExit(int) { std::abort(); } + +} // namespace Common + int main() { - TestMappedSrtUsesDirectReaderByDefault(); - TestUnbasedFlatCacheHitFailsClosed(); - std::puts("ShaderStageRuntimeTests: all cases passed"); - return 0; + TestMappedSrtUsesDirectReaderByDefault(); + TestUnbasedFlatCacheHitFailsClosed(); + std::puts("ShaderStageRuntimeTests: all cases passed"); + return 0; } + +// Keep this focused standalone target self-contained by amalgamating its small +// typed-IR implementation set. +#include "graphics/shader/recompiler/ir/Block.cpp" +#include "graphics/shader/recompiler/ir/Type.cpp" +#include "graphics/shader/recompiler/ir/Value.cpp" +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" +#include "graphics/shader/recompiler/ir/ValueProgram.cpp" diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 194c54251..bd00fa899 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -11,16 +11,20 @@ #include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h" #include "graphics/shader/recompiler/ExecMask.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" -#include "graphics/shader/recompiler/cfg/ShaderCFG.h" -#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h" -#include "graphics/shader/recompiler/emitter/SpirvEmitter.h" -#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h" -#include "graphics/shader/recompiler/ir/ResourceTracking.h" -#include "graphics/shader/recompiler/ir/ScalarProvenance.h" +#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" +#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" +#include "graphics/shader/recompiler/frontend/translate/Translate.h" +#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" +#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" +#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h" +#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h" +#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h" +#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h" -#include "graphics/shader/recompiler/ir/SrtPatcher.h" -#include "graphics/shader/recompiler/ir/SrtWalker.h" +#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h" +#include "graphics/shader/recompiler/ir/passes/SrtWalker.h" +#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h" +#include "graphics/shader/recompiler/ir/ValueProgram.h" #include "graphics/shader/shader.h" #include "libs/agc.h" #include "spirv-tools/libspirv.hpp" @@ -40,7735 +44,8864 @@ namespace Libs::Graphics { namespace { -void Check(bool value, const char* text) { - if (!value) { - std::fprintf(stderr, "ShaderCfgTests: failed: %s\n", text); - std::abort(); - } +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "ShaderCfgTests: failed: %s\n", text); + std::abort(); + } } -void CheckSpirvBinaryValidates(const std::vector& binary) { - spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); - std::string messages; - - tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, - const spv_position_t& position, const char* message) { - char buffer[1024] = {}; - std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, position.column, - message); - messages += buffer; - }); - - if (!tools.Validate(binary)) { - std::fprintf(stderr, "SPIR-V binary validation failed:\n%s\n", messages.c_str()); - std::abort(); - } +std::vector +CfgInstructionCoverage(const ShaderRecompiler::CFG::Graph &graph, + size_t instruction_count) { + std::vector coverage(instruction_count); + for (const auto &block : graph.blocks) { + Check(block.inst_begin <= block.inst_end && + block.inst_end <= instruction_count, + "CFG block has an invalid instruction range"); + for (size_t index = block.inst_begin; index < block.inst_end; index++) { + coverage[index]++; + } + } + return coverage; } -std::string DisassembleSpirvBinary(const std::vector& binary) { - spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); - std::string messages; - std::string source; +void CheckSpirvBinaryValidates(const std::vector &binary) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string messages; - tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, - const spv_position_t& position, const char* message) { - char buffer[1024] = {}; - std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, position.column, - message); - messages += buffer; - }); + tools.SetMessageConsumer([&messages](spv_message_level_t, const char *, + const spv_position_t &position, + const char *message) { + char buffer[1024] = {}; + std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, + position.column, message); + messages += buffer; + }); - if (!tools.Disassemble(binary.data(), binary.size(), &source, - SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | - SPV_BINARY_TO_TEXT_OPTION_INDENT)) { - std::fprintf(stderr, "SPIR-V binary disassembly failed:\n%s\n", messages.c_str()); - std::abort(); - } - return std::string(source.c_str()); + if (!tools.Validate(binary)) { + std::fprintf(stderr, "SPIR-V binary validation failed:\n%s\n", + messages.c_str()); + std::abort(); + } } -uint32_t CountSourceOccurrences(const std::string& source, const char* needle) { - uint32_t count = 0; - uint32_t from = 0; - for (;;) { - const auto found = Common::FindIndex(source, std::string(needle), from); - if (found == Common::FIND_INVALID_INDEX) { - return count; - } - count++; - from = found + static_cast(std::strlen(needle)); - } +std::string DisassembleSpirvBinary(const std::vector &binary) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); + std::string messages; + std::string source; + + tools.SetMessageConsumer([&messages](spv_message_level_t, const char *, + const spv_position_t &position, + const char *message) { + char buffer[1024] = {}; + std::snprintf(buffer, sizeof(buffer), "%zu:%zu: %s\n", position.line, + position.column, message); + messages += buffer; + }); + + if (!tools.Disassemble(binary.data(), binary.size(), &source, + SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES | + SPV_BINARY_TO_TEXT_OPTION_INDENT)) { + std::fprintf(stderr, "SPIR-V binary disassembly failed:\n%s\n", + messages.c_str()); + std::abort(); + } + return std::string(source.c_str()); } -bool SpirvSourceHasInstructionUsing(const std::string& source, const char* opcode, - const char* name) { - std::istringstream stream(source); - std::string line; - while (std::getline(stream, line)) { - if (Common::ContainsStr(line, opcode) && Common::ContainsStr(line, name)) { - return true; - } - } - return false; +uint32_t CountSourceOccurrences(const std::string &source, const char *needle) { + uint32_t count = 0; + uint32_t from = 0; + for (;;) { + const auto found = Common::FindIndex(source, std::string(needle), from); + if (found == Common::FIND_INVALID_INDEX) { + return count; + } + count++; + from = found + static_cast(std::strlen(needle)); + } } -bool SpirvContainsOpcode(const std::vector& binary, uint32_t opcode) { - for (uint32_t word: binary) { - if ((word & 0xffffu) == opcode) { - return true; - } - } - return false; +bool SpirvSourceHasInstructionUsing(const std::string &source, + const char *opcode, const char *name) { + std::istringstream stream(source); + std::string line; + while (std::getline(stream, line)) { + if (Common::ContainsStr(line, opcode) && Common::ContainsStr(line, name)) { + return true; + } + } + return false; } -uint32_t SpirvInstructionOpcodeCount(const std::vector& binary, uint32_t opcode) { - uint32_t count = 0; - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t op = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return count; - } - if (op == opcode) { - count++; - } - i += word_count; - } - return count; +bool SpirvContainsOpcode(const std::vector &binary, uint32_t opcode) { + for (uint32_t word : binary) { + if ((word & 0xffffu) == opcode) { + return true; + } + } + return false; } -bool SpirvContainsVectorShuffle(const std::vector& binary, - const std::array& selectors) { - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t op = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return false; - } - if (op == 79u && word_count == 9u && - std::equal(selectors.begin(), selectors.end(), binary.begin() + i + 5u)) { - return true; - } - i += word_count; - } - return false; +uint32_t SpirvInstructionOpcodeCount(const std::vector &binary, + uint32_t opcode) { + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t op = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (op == opcode) { + count++; + } + i += word_count; + } + return count; } -bool SpirvContainsCapability(const std::vector& binary, uint32_t capability) { - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return false; - } - if (opcode == 17u && word_count >= 2u && binary[i + 1] == capability) { - return true; - } - i += word_count; - } - return false; +bool SpirvSourceHasInstructionOperand(const std::string &source, + const char *opcode, + const char *operand) { + std::istringstream lines(source); + std::string line; + while (std::getline(lines, line)) { + std::istringstream words(line); + std::string word; + bool instruction = false; + while (words >> word) { + instruction |= word == opcode; + if (instruction && word == operand) { + return true; + } + } + } + return false; } -bool SpirvContainsExecutionMode(const std::vector& binary, uint32_t mode) { - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return false; - } - if (opcode == 16u && word_count >= 3u && binary[i + 2] == mode) { - return true; - } - i += word_count; - } - return false; +bool SpirvContainsVectorShuffle(const std::vector &binary, + const std::array &selectors) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t op = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (op == 79u && word_count == 9u && + std::equal(selectors.begin(), selectors.end(), + binary.begin() + i + 5u)) { + return true; + } + i += word_count; + } + return false; } -bool SpirvContainsExtInst(const std::vector& binary, uint32_t ext_inst) { - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return false; - } - if (opcode == 12u && word_count >= 5u && binary[i + 4] == ext_inst) { - return true; - } - i += word_count; - } - return false; +bool SpirvContainsCapability(const std::vector &binary, + uint32_t capability) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 17u && word_count >= 2u && binary[i + 1] == capability) { + return true; + } + i += word_count; + } + return false; } -uint32_t SpirvExtInstCount(const std::vector& binary, uint32_t ext_inst) { - uint32_t count = 0; - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return count; - } - if (opcode == 12u && word_count >= 5u && binary[i + 4] == ext_inst) { - count++; - } - i += word_count; - } - return count; +bool SpirvContainsExecutionMode(const std::vector &binary, + uint32_t mode) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 16u && word_count >= 3u && binary[i + 2] == mode) { + return true; + } + i += word_count; + } + return false; } -bool SpirvContainsTypeImage(const std::vector& binary, uint32_t dim, uint32_t arrayed, - uint32_t sampled, uint32_t multisampled = 0) { - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return false; - } - if (opcode == 25u && word_count >= 9u && binary[i + 3] == dim && binary[i + 5] == arrayed && - binary[i + 6] == multisampled && binary[i + 7] == sampled) { - return true; - } - i += word_count; - } - return false; +bool SpirvContainsExtInst(const std::vector &binary, + uint32_t ext_inst) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 12u && word_count >= 5u && binary[i + 4] == ext_inst) { + return true; + } + i += word_count; + } + return false; } -uint32_t SpirvDecorationValueCount(const std::vector& binary, uint32_t decoration, - uint32_t value) { - uint32_t count = 0; - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return count; - } - if (opcode == 71u && word_count >= 4u && binary[i + 2] == decoration && - binary[i + 3] == value) { - count++; - } - i += word_count; - } - return count; +uint32_t SpirvExtInstCount(const std::vector &binary, + uint32_t ext_inst) { + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (opcode == 12u && word_count >= 5u && binary[i + 4] == ext_inst) { + count++; + } + i += word_count; + } + return count; } -bool SpirvHasDecorationValue(const std::vector& binary, uint32_t decoration, - uint32_t value) { - return SpirvDecorationValueCount(binary, decoration, value) != 0; +bool SpirvContainsTypeImage(const std::vector &binary, uint32_t dim, + uint32_t arrayed, uint32_t sampled, + uint32_t multisampled = 0) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 25u && word_count >= 9u && binary[i + 3] == dim && + binary[i + 5] == arrayed && binary[i + 6] == multisampled && + binary[i + 7] == sampled) { + return true; + } + i += word_count; + } + return false; } -std::vector SpirvDecorationValueTargets(const std::vector& binary, - uint32_t decoration, uint32_t value) { - std::vector targets; - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return targets; - } - if (opcode == 71u && word_count >= 4u && binary[i + 2] == decoration && - binary[i + 3] == value) { - targets.push_back(binary[i + 1]); - } - i += word_count; - } - return targets; +uint32_t SpirvDecorationValueCount(const std::vector &binary, + uint32_t decoration, uint32_t value) { + uint32_t count = 0; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return count; + } + if (opcode == 71u && word_count >= 4u && binary[i + 2] == decoration && + binary[i + 3] == value) { + count++; + } + i += word_count; + } + return count; } -bool SpirvTargetHasDecoration(const std::vector& binary, uint32_t target, - uint32_t decoration) { - for (size_t i = 5; i < binary.size();) { - const uint32_t word = binary[i]; - const uint32_t opcode = word & 0xffffu; - const uint32_t word_count = word >> 16u; - if (word_count == 0 || i + word_count > binary.size()) { - return false; - } - if (opcode == 71u && word_count >= 3u && binary[i + 1] == target && - binary[i + 2] == decoration) { - return true; - } - i += word_count; - } - return false; +bool SpirvHasDecorationValue(const std::vector &binary, + uint32_t decoration, uint32_t value) { + return SpirvDecorationValueCount(binary, decoration, value) != 0; } -bool SpirvHasDecorationValueWithDecoration(const std::vector& binary, - uint32_t value_decoration, uint32_t value, +std::vector +SpirvDecorationValueTargets(const std::vector &binary, + uint32_t decoration, uint32_t value) { + std::vector targets; + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return targets; + } + if (opcode == 71u && word_count >= 4u && binary[i + 2] == decoration && + binary[i + 3] == value) { + targets.push_back(binary[i + 1]); + } + i += word_count; + } + return targets; +} + +bool SpirvTargetHasDecoration(const std::vector &binary, + uint32_t target, uint32_t decoration) { + for (size_t i = 5; i < binary.size();) { + const uint32_t word = binary[i]; + const uint32_t opcode = word & 0xffffu; + const uint32_t word_count = word >> 16u; + if (word_count == 0 || i + word_count > binary.size()) { + return false; + } + if (opcode == 71u && word_count >= 3u && binary[i + 1] == target && + binary[i + 2] == decoration) { + return true; + } + i += word_count; + } + return false; +} + +bool SpirvHasDecorationValueWithDecoration(const std::vector &binary, + uint32_t value_decoration, + uint32_t value, uint32_t decoration) { - const auto targets = SpirvDecorationValueTargets(binary, value_decoration, value); - return std::any_of(targets.begin(), targets.end(), [&](uint32_t target) { - return SpirvTargetHasDecoration(binary, target, decoration); - }); + const auto targets = + SpirvDecorationValueTargets(binary, value_decoration, value); + return std::any_of(targets.begin(), targets.end(), [&](uint32_t target) { + return SpirvTargetHasDecoration(binary, target, decoration); + }); } -bool ProgramHasInput(const ShaderRecompiler::IR::Program& program, +bool ProgramHasInput(const ShaderRecompiler::IR::Program &program, ShaderRecompiler::IR::StageInputKind kind) { - return std::any_of(program.info.inputs.begin(), program.info.inputs.end(), - [kind](const auto& input) { return input.kind == kind; }); + return std::any_of(program.info.inputs.begin(), program.info.inputs.end(), + [kind](const auto &input) { return input.kind == kind; }); } -uint32_t ProgramInputCount(const ShaderRecompiler::IR::Program& program, +uint32_t ProgramInputCount(const ShaderRecompiler::IR::Program &program, ShaderRecompiler::IR::StageInputKind kind) { - return static_cast( - std::count_if(program.info.inputs.begin(), program.info.inputs.end(), - [kind](const auto& input) { return input.kind == kind; })); + return static_cast( + std::count_if(program.info.inputs.begin(), program.info.inputs.end(), + [kind](const auto &input) { return input.kind == kind; })); } ShaderComputeInputInfo RegressionComputeInputInfo() { - ShaderComputeInputInfo input_info; - input_info.threads_num[0] = 1; - input_info.threads_num[1] = 1; - input_info.threads_num[2] = 1; - input_info.workgroup_register = 40; - return input_info; + ShaderComputeInputInfo input_info; + input_info.threads_num[0] = 1; + input_info.threads_num[1] = 1; + input_info.threads_num[2] = 1; + input_info.workgroup_register = 40; + return input_info; } -ShaderPixelInputInfo RegressionPixelInputInfo() { - return {}; -} +ShaderPixelInputInfo RegressionPixelInputInfo() { return {}; } -void SetIdentityInterpolatorSettings(ShaderPixelInputInfo* input_info) { - Check(input_info != nullptr, "invalid pixel input info"); - for (uint32_t i = 0; i < std::size(input_info->interpolator_settings); i++) { - input_info->interpolator_settings[i] = i; - } +void SetIdentityInterpolatorSettings(ShaderPixelInputInfo *input_info) { + Check(input_info != nullptr, "invalid pixel input info"); + for (uint32_t i = 0; i < std::size(input_info->interpolator_settings); i++) { + input_info->interpolator_settings[i] = i; + } } void EnsureConfigInitialized() { - static bool config_initialized = false; - if (!config_initialized) { - static Common::Subsystems subsystems; - Common::InitializeThreads(); - subsystems.Initialize(); - Config::ConfigOptions options; - options.printf_direction = Config::OutputDirection::Silent; - Config::Load(options); - subsystems.Initialize(); - ShaderInit(); - config_initialized = true; - } + static bool config_initialized = false; + if (!config_initialized) { + static Common::Subsystems subsystems; + Common::InitializeThreads(); + subsystems.Initialize(); + Config::ConfigOptions options; + options.printf_direction = Config::OutputDirection::Silent; + Config::Load(options); + subsystems.Initialize(); + ShaderInit(); + config_initialized = true; + } } void TestResourceDescriptorClassification() { - uint32_t raw_texture[8] = {}; - raw_texture[3] = 9u << 28u; - raw_texture[5] = 2u << 27u; - Check(ShaderClassifyResourceDescriptor(raw_texture) == ResourceDescriptorType::Texture, - "raw texture descriptor must not be classified by ResourceDescriptor byte 23 fields"); + uint32_t raw_texture[8] = {}; + raw_texture[3] = 9u << 28u; + raw_texture[5] = 2u << 27u; + Check(ShaderClassifyResourceDescriptor(raw_texture) == + ResourceDescriptorType::Texture, + "raw texture descriptor must not be classified by ResourceDescriptor " + "byte 23 fields"); - uint32_t tagged_buffer[8] = {}; - tagged_buffer[5] = 1u << 27u; - Check(ShaderClassifyResourceDescriptor(tagged_buffer) == ResourceDescriptorType::Buffer, - "tagged ResourceDescriptor buffer was not recognized"); + uint32_t tagged_buffer[8] = {}; + tagged_buffer[5] = 1u << 27u; + Check(ShaderClassifyResourceDescriptor(tagged_buffer) == + ResourceDescriptorType::Buffer, + "tagged ResourceDescriptor buffer was not recognized"); - uint32_t tagged_sampler[8] = {}; - tagged_sampler[5] = 2u << 27u; - Check(ShaderClassifyResourceDescriptor(tagged_sampler) == ResourceDescriptorType::Sampler, - "tagged ResourceDescriptor sampler was not recognized"); + uint32_t tagged_sampler[8] = {}; + tagged_sampler[5] = 2u << 27u; + Check(ShaderClassifyResourceDescriptor(tagged_sampler) == + ResourceDescriptorType::Sampler, + "tagged ResourceDescriptor sampler was not recognized"); - uint32_t tagged_unused[8] = {}; - tagged_unused[5] = 3u << 27u; - Check(ShaderClassifyResourceDescriptor(tagged_unused) == ResourceDescriptorType::Unused, - "tagged ResourceDescriptor unused slot was not recognized"); + uint32_t tagged_unused[8] = {}; + tagged_unused[5] = 3u << 27u; + Check(ShaderClassifyResourceDescriptor(tagged_unused) == + ResourceDescriptorType::Unused, + "tagged ResourceDescriptor unused slot was not recognized"); - uint32_t raw_buffer[8] = {}; - Check(ShaderClassifyResourceDescriptor(raw_buffer) == ResourceDescriptorType::Buffer, - "raw buffer descriptor fallback was not preserved"); + uint32_t raw_buffer[8] = {}; + Check(ShaderClassifyResourceDescriptor(raw_buffer) == + ResourceDescriptorType::Buffer, + "raw buffer descriptor fallback was not preserved"); } constexpr uint32_t EncodeSMovB32(uint32_t dst, uint32_t src) { - return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) | (src & 0xffu); + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | (0x03u << 8u) | + (src & 0xffu); } constexpr uint32_t EncodeSop1(uint32_t opcode, uint32_t dst, uint32_t src) { - return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | ((opcode & 0xffu) << 8u) | - (src & 0xffu); + return 0x80000000u | (0x7du << 23u) | ((dst & 0x7fu) << 16u) | + ((opcode & 0xffu) << 8u) | (src & 0xffu); } -constexpr uint32_t EncodeSop2(uint32_t opcode, uint32_t dst, uint32_t src0, uint32_t src1) { - return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | - ((src1 & 0xffu) << 8u) | (src0 & 0xffu); +constexpr uint32_t EncodeSop2(uint32_t opcode, uint32_t dst, uint32_t src0, + uint32_t src1) { + return 0x80000000u | ((opcode & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); } constexpr uint32_t EncodeSopk(uint32_t opcode, uint32_t dst, int16_t imm) { - return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) | ((dst & 0x7fu) << 16u) | - static_cast(imm); + return 0x80000000u | (((opcode + 0x60u) & 0x7fu) << 23u) | + ((dst & 0x7fu) << 16u) | static_cast(imm); } constexpr uint32_t EncodeSopc(uint32_t opcode, uint32_t src0, uint32_t src1) { - return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) | ((src1 & 0xffu) << 8u) | - (src0 & 0xffu); + return 0x80000000u | (0x7eu << 23u) | ((opcode & 0x7fu) << 16u) | + ((src1 & 0xffu) << 8u) | (src0 & 0xffu); } constexpr uint32_t EncodeSopp(uint32_t opcode, uint32_t simm = 0) { - return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) | (simm & 0xffffu); + return 0x80000000u | (0x7fu << 23u) | ((opcode & 0x7fu) << 16u) | + (simm & 0xffffu); } void TestNativeShaderResourceDependencies() { - const auto stages = - ShaderPipelineStages(vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment | - vk::ShaderStageFlagBits::eCompute); - Check(stages == (vk::PipelineStageFlagBits::eVertexShader | - vk::PipelineStageFlagBits::eFragmentShader | - vk::PipelineStageFlagBits::eComputeShader), - "shader-stage dependency mapping was not exact"); - const auto shader_barrier = MakeShaderWriteDependency(); - Check(shader_barrier.srcAccessMask == vk::AccessFlagBits::eShaderWrite && - (shader_barrier.dstAccessMask & vk::AccessFlagBits::eShaderRead) && - (shader_barrier.dstAccessMask & vk::AccessFlagBits::eVertexAttributeRead), - "shader-write dependency does not expose draw/dispatch buffer writes"); + const auto stages = ShaderPipelineStages(vk::ShaderStageFlagBits::eVertex | + vk::ShaderStageFlagBits::eFragment | + vk::ShaderStageFlagBits::eCompute); + Check(stages == (vk::PipelineStageFlagBits::eVertexShader | + vk::PipelineStageFlagBits::eFragmentShader | + vk::PipelineStageFlagBits::eComputeShader), + "shader-stage dependency mapping was not exact"); + const auto shader_barrier = MakeShaderWriteDependency(); + Check(shader_barrier.srcAccessMask == vk::AccessFlagBits::eShaderWrite && + (shader_barrier.dstAccessMask & vk::AccessFlagBits::eShaderRead) && + (shader_barrier.dstAccessMask & + vk::AccessFlagBits::eVertexAttributeRead), + "shader-write dependency does not expose draw/dispatch buffer writes"); - ShaderRecompiler::IR::Program program; - program.info.buffers.resize(3); - program.info.buffers[0].written = true; - program.info.buffers[1].read = true; - program.info.buffers[2].written = true; - ShaderRecompiler::IR::ResourceSnapshot resources; - resources.buffers.resize(3); - auto set_buffer = [&](uint32_t index, uint64_t address, uint16_t stride, uint32_t records) { - ShaderBufferResource descriptor; - descriptor.UpdateAddress48(address); - descriptor.fields[1] |= static_cast(stride) << 16u; - descriptor.fields[2] = records; - std::memcpy(resources.buffers[index].dwords.data(), descriptor.fields, - sizeof(descriptor.fields)); - resources.buffers[index].dword_count = 4; - }; - set_buffer(0, 0x1000, 16, 3); - set_buffer(1, 0x1800, 4, 7); - set_buffer(2, 0x2000, 0, 64); - const auto writes = CollectShaderBufferWrites(program, resources); - Check(writes == std::vector({{0x1000, 48}, {0x2000, 64}}), - "graphics/compute write collector lost or invented a storage-buffer range"); + ShaderRecompiler::IR::Program program; + program.info.buffers.resize(3); + program.info.buffers[0].written = true; + program.info.buffers[1].read = true; + program.info.buffers[2].written = true; + ShaderRecompiler::IR::ResourceSnapshot resources; + resources.buffers.resize(3); + auto set_buffer = [&](uint32_t index, uint64_t address, uint16_t stride, + uint32_t records) { + ShaderBufferResource descriptor; + descriptor.UpdateAddress48(address); + descriptor.fields[1] |= static_cast(stride) << 16u; + descriptor.fields[2] = records; + std::memcpy(resources.buffers[index].dwords.data(), descriptor.fields, + sizeof(descriptor.fields)); + resources.buffers[index].dword_count = 4; + }; + set_buffer(0, 0x1000, 16, 3); + set_buffer(1, 0x1800, 4, 7); + set_buffer(2, 0x2000, 0, 64); + const auto writes = CollectShaderBufferWrites(program, resources); + Check(writes == + std::vector({{0x1000, 48}, {0x2000, 64}}), + "graphics/compute write collector lost or invented a storage-buffer " + "range"); - VulkanBuffer buffer; - buffer.buffer = reinterpret_cast(uintptr_t {1}); - const auto gds_barrier = MakeGdsDependency(buffer.buffer); - Check((gds_barrier.srcAccessMask & vk::AccessFlagBits::eHostWrite) && - (gds_barrier.srcAccessMask & vk::AccessFlagBits::eTransferWrite) && - (gds_barrier.srcAccessMask & vk::AccessFlagBits::eShaderWrite) && - gds_barrier.dstAccessMask == - (vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eShaderWrite) && - gds_barrier.size == VK_WHOLE_SIZE, - "GDS dependency does not order host/transfer/shader writes"); + VulkanBuffer buffer; + buffer.buffer = reinterpret_cast(uintptr_t{1}); + const auto gds_barrier = MakeGdsDependency(buffer.buffer); + Check((gds_barrier.srcAccessMask & vk::AccessFlagBits::eHostWrite) && + (gds_barrier.srcAccessMask & vk::AccessFlagBits::eTransferWrite) && + (gds_barrier.srcAccessMask & vk::AccessFlagBits::eShaderWrite) && + gds_barrier.dstAccessMask == (vk::AccessFlagBits::eShaderRead | + vk::AccessFlagBits::eShaderWrite) && + gds_barrier.size == VK_WHOLE_SIZE, + "GDS dependency does not order host/transfer/shader writes"); } void TestNormalizedImageContracts() { - ImageInfo container {}; - container.data = {0x10000, 0x15000}; - container.pixel_format = vk::Format::eR8G8B8A8Unorm; - container.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; - container.type = Prospero::ImageType::kColor2D; - container.extent = {64, 64, 1}; - container.resources = {3, 4}; - container.pitch = 64; - container.bytes_per_block = 4; - container.samples = 1; - container.tile_mode = Prospero::TileMode::kStandard64KB; - container.mip_layout[0] = {0, 0x10000, 64, 64}; - container.mip_layout[1] = {0x10000, 0x4000, 32, 32}; - container.mip_layout[2] = {0x14000, 0x1000, 16, 16}; + ImageInfo container{}; + container.data = {0x10000, 0x15000}; + container.pixel_format = vk::Format::eR8G8B8A8Unorm; + container.guest_format = Prospero::BufferFormat::k8_8_8_8UNorm; + container.type = Prospero::ImageType::kColor2D; + container.extent = {64, 64, 1}; + container.resources = {3, 4}; + container.pitch = 64; + container.bytes_per_block = 4; + container.samples = 1; + container.tile_mode = Prospero::TileMode::kStandard64KB; + container.mip_layout[0] = {0, 0x10000, 64, 64}; + container.mip_layout[1] = {0x10000, 0x4000, 32, 32}; + container.mip_layout[2] = {0x14000, 0x1000, 16, 16}; - ImageInfo subresource = container; - subresource.data = {0x22000, 0x1000}; - subresource.extent = {32, 32, 1}; - subresource.resources = {1, 1}; - subresource.pitch = 32; - subresource.mip_layout = {}; - subresource.mip_layout[0] = {0, 0x1000, 32, 32}; + ImageInfo subresource = container; + subresource.data = {0x22000, 0x1000}; + subresource.extent = {32, 32, 1}; + subresource.resources = {1, 1}; + subresource.pitch = 32; + subresource.mip_layout = {}; + subresource.mip_layout[0] = {0, 0x1000, 32, 32}; - Check(container.BlockExtent() == vk::Extent2D {64, 64}, - "normalized image block extent changed"); - Check(subresource.IsCompatible(container), "normalized compatible image was rejected"); - Check(subresource.MipOf(container) == 1, "normalized mip lookup missed a subresource"); - Check(subresource.SliceOf(container, 1) == 2, "normalized slice lookup missed a subresource"); + Check(container.BlockExtent() == vk::Extent2D{64, 64}, + "normalized image block extent changed"); + Check(subresource.IsCompatible(container), + "normalized compatible image was rejected"); + Check(subresource.MipOf(container) == 1, + "normalized mip lookup missed a subresource"); + Check(subresource.SliceOf(container, 1) == 2, + "normalized slice lookup missed a subresource"); - auto incompatible = subresource; - incompatible.samples = 2; - Check(!incompatible.IsCompatible(container) && incompatible.MipOf(container) == -1, - "sample-count mismatch was accepted as a compatible image"); + auto incompatible = subresource; + incompatible.samples = 2; + Check(!incompatible.IsCompatible(container) && + incompatible.MipOf(container) == -1, + "sample-count mismatch was accepted as a compatible image"); - auto compressed = container; - compressed.guest_format = Prospero::BufferFormat::kBc3UNorm; - compressed.pitch = 128; - compressed.extent.height = 64; - Check(compressed.BlockExtent() == vk::Extent2D {32, 16}, - "block-compressed extent was not expressed in blocks"); + auto compressed = container; + compressed.guest_format = Prospero::BufferFormat::kBc3UNorm; + compressed.pitch = 128; + compressed.extent.height = 64; + Check(compressed.BlockExtent() == vk::Extent2D{32, 16}, + "block-compressed extent was not expressed in blocks"); - Check(ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm, vk::Format::eR8G8B8A8Uint) && - !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat, vk::Format::eR32Sfloat) && - ImageViewOps::FormatsCompatible(vk::Format::eBc3UnormBlock, - vk::Format::eR32G32B32A32Uint), - "Vulkan image-view compatibility classes diverged from production"); + Check(ImageViewOps::FormatsCompatible(vk::Format::eR8G8B8A8Unorm, + vk::Format::eR8G8B8A8Uint) && + !ImageViewOps::FormatsCompatible(vk::Format::eD32Sfloat, + vk::Format::eR32Sfloat) && + ImageViewOps::FormatsCompatible(vk::Format::eBc3UnormBlock, + vk::Format::eR32G32B32A32Uint), + "Vulkan image-view compatibility classes diverged from production"); } void TestNativeSubgroupPolicy() { - GraphicContext context; - context.subgroup_size = 32; - context.min_subgroup_size = 32; - context.max_subgroup_size = 32; - context.subgroup_size_control_enabled = true; - context.required_subgroup_size_stages = vk::ShaderStageFlagBits::eAll; - ShaderRecompiler::IR::Program safe; - safe.wave_size = 32; - safe.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eVertex, safe) - .mode == ShaderSubgroupMode::Natural, - "native wave32 policy changed"); - safe.wave_size = 64; - Check(SelectGraphicsLaneMaskMode(safe.wave_size) == ShaderLaneMaskMode::PerInvocation && - ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eVertex, safe) - .mode == ShaderSubgroupMode::Unsupported, - "wave64 graphics mismatch accepted native-wave mask lowering"); - safe.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eVertex, safe) - .mode == ShaderSubgroupMode::PerInvocationGraphics, - "wave64 graphics mismatch did not select per-invocation masks"); + const auto SetTypedValues = [](ShaderRecompiler::IR::Program &program, + ShaderRecompiler::IR::ValueOpcode opcode = + ShaderRecompiler::IR::ValueOpcode::Void) { + using namespace ShaderRecompiler::IR; + program.values = std::make_shared(); + program.values->block_storage.push_back(std::make_unique()); + auto *block = program.values->block_storage.back().get(); + program.values->blocks.push_back(block); + if (opcode == ValueOpcode::ReadLane) { + block->AppendNewInst(opcode, {Value(0u), Value(0u)}); + } else if (opcode == ValueOpcode::Ballot) { + block->AppendNewInst(opcode, {Value(true)}); + } else if (opcode == ValueOpcode::DataAppend) { + auto &resource = block->AppendNewInst(ValueOpcode::GetLdsResource); + block->AppendNewInst(opcode, {Value(&resource), Value(0u), Value(true), + Value(1u), Value(0u)}); + } + }; - ShaderRecompiler::IR::Program cross_lane = safe; - cross_lane.blocks.emplace_back().instructions.emplace_back().op = - ShaderRecompiler::IR::Opcode::ReadLaneU32; - Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(cross_lane) && - ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eVertex, cross_lane) - .mode == ShaderSubgroupMode::PerInvocationGraphics, - "graphics mismatch did not select per-invocation masks"); - auto cross_lane_compute = cross_lane; - cross_lane_compute.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eCompute, cross_lane_compute) - .mode == ShaderSubgroupMode::Unsupported, - "cross-lane compute mismatch bypassed the exact subgroup requirement"); + GraphicContext context; + context.subgroup_size = 32; + context.min_subgroup_size = 32; + context.max_subgroup_size = 32; + context.subgroup_size_control_enabled = true; + context.required_subgroup_size_stages = vk::ShaderStageFlagBits::eAll; + ShaderRecompiler::IR::Program safe; + safe.wave_size = 32; + safe.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + SetTypedValues(safe); + Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eVertex, safe) + .mode == ShaderSubgroupMode::Natural, + "native wave32 policy changed"); + safe.wave_size = 64; + Check(SelectGraphicsLaneMaskMode(safe.wave_size) == + ShaderLaneMaskMode::PerInvocation && + ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eVertex, safe) + .mode == ShaderSubgroupMode::Unsupported, + "wave64 graphics mismatch accepted native-wave mask lowering"); + safe.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eVertex, safe) + .mode == ShaderSubgroupMode::PerInvocationGraphics, + "wave64 graphics mismatch did not select per-invocation masks"); - ShaderRecompiler::IR::Program zero_exec = safe; - zero_exec.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - zero_exec.provenance.values.resize(3); - zero_exec.provenance.values[2].op = ShaderRecompiler::IR::ScalarValueOp::Constant; - zero_exec.provenance.values[2].imm = 0; - auto& zero_bfm = zero_exec.blocks.emplace_back().instructions.emplace_back(); - zero_bfm.op = ShaderRecompiler::IR::Opcode::BitFieldMaskU64; - zero_bfm.dst.kind = ShaderRecompiler::IR::OperandKind::Register; - zero_bfm.dst.reg = {ShaderRecompiler::IR::RegisterFile::Exec, 0}; - zero_bfm.src_count = 2; - zero_bfm.scalar_sources[0] = 2; - Check(!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(zero_exec) && - ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eCompute, zero_exec) - .mode == ShaderSubgroupMode::FlattenedMasks, - "compile-time uniform-zero EXEC write did not stay on the mask-free path"); + ShaderRecompiler::IR::Program cross_lane = safe; + SetTypedValues(cross_lane, ShaderRecompiler::IR::ValueOpcode::ReadLane); + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(cross_lane) && + ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eVertex, + cross_lane) + .mode == ShaderSubgroupMode::PerInvocationGraphics, + "graphics mismatch did not select per-invocation masks"); + auto cross_lane_compute = cross_lane; + cross_lane_compute.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eCompute, + cross_lane_compute) + .mode == ShaderSubgroupMode::Unsupported, + "cross-lane compute mismatch bypassed the exact subgroup requirement"); - ShaderRecompiler::IR::Program selective_exec = safe; - auto& move_exec = selective_exec.blocks.emplace_back().instructions.emplace_back(); - move_exec.op = ShaderRecompiler::IR::Opcode::MoveU32; - move_exec.dst.kind = ShaderRecompiler::IR::OperandKind::Register; - move_exec.dst.reg = {ShaderRecompiler::IR::RegisterFile::Exec, 0}; - Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(selective_exec), - "selective EXEC write was classified mask-free"); + ShaderRecompiler::IR::Program zero_exec = safe; + zero_exec.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + SetTypedValues(zero_exec); + Check(!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(zero_exec) && + ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eCompute, + zero_exec) + .mode == ShaderSubgroupMode::FlattenedMasks, + "compile-time uniform-zero EXEC write did not stay on the mask-free " + "path"); - ShaderRecompiler::IR::Program carry = safe; - auto& add_carry = carry.blocks.emplace_back().instructions.emplace_back(); - add_carry.op = ShaderRecompiler::IR::Opcode::IAddCarryU32; - add_carry.dst2.kind = ShaderRecompiler::IR::OperandKind::Register; - add_carry.dst2.reg = {ShaderRecompiler::IR::RegisterFile::Vcc, 0}; - Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(carry), - "lane-varying VCC carry producer was classified mask-free"); + ShaderRecompiler::IR::Program selective_exec = safe; + SetTypedValues(selective_exec, ShaderRecompiler::IR::ValueOpcode::Ballot); + Check( + ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(selective_exec), + "selective EXEC write was classified mask-free"); - ShaderRecompiler::IR::Program vcc_branch = safe; - auto& branch_block = vcc_branch.blocks.emplace_back(); - branch_block.terminator.kind = ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch; - branch_block.terminator.condition = ShaderRecompiler::CFG::BranchCondition::VccNonZero; - Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(vcc_branch), - "whole-wave VCC branch was classified mask-free"); + ShaderRecompiler::IR::Program carry = safe; + SetTypedValues(carry, ShaderRecompiler::IR::ValueOpcode::Ballot); + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(carry), + "lane-varying VCC carry producer was classified mask-free"); - ShaderRecompiler::IR::Program ds_partial = safe; - ds_partial.lane_mask_mode = ShaderLaneMaskMode::NativeWave; - ds_partial.blocks.emplace_back().instructions.emplace_back().op = - ShaderRecompiler::IR::Opcode::DsAppend; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eCompute, ds_partial) - .mode == ShaderSubgroupMode::Unsupported, - "partial wave64 DS append bypassed the exact subgroup requirement"); - context.max_subgroup_size = 64; - const auto controlled = - ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eCompute, cross_lane_compute); - Check(controlled.mode == ShaderSubgroupMode::Controlled && controlled.required_size == 64, - "supported controlled wave64 was not preferred over splitting"); - context.subgroup_size = 64; - context.subgroup_size_control_enabled = false; - cross_lane.wave_size = 32; - cross_lane.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eFragment, cross_lane) - .mode == ShaderSubgroupMode::Unsupported, - "inverse graphics mismatch was accepted as one guest wave"); - cross_lane_compute.wave_size = 32; - Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities {context}, - vk::ShaderStageFlagBits::eCompute, cross_lane_compute) - .mode == ShaderSubgroupMode::Unsupported, - "inverse cross-lane compute mismatch was accepted"); + ShaderRecompiler::IR::Program vcc_branch = safe; + SetTypedValues(vcc_branch, ShaderRecompiler::IR::ValueOpcode::Ballot); + Check(ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(vcc_branch), + "whole-wave VCC branch was classified mask-free"); + + ShaderRecompiler::IR::Program ds_partial = safe; + ds_partial.lane_mask_mode = ShaderLaneMaskMode::NativeWave; + SetTypedValues(ds_partial, ShaderRecompiler::IR::ValueOpcode::DataAppend); + Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eCompute, ds_partial) + .mode == ShaderSubgroupMode::Unsupported, + "partial wave64 DS append bypassed the exact subgroup requirement"); + context.max_subgroup_size = 64; + const auto controlled = ConfigureShaderSubgroup( + ShaderSubgroupCapabilities{context}, vk::ShaderStageFlagBits::eCompute, + cross_lane_compute); + Check(controlled.mode == ShaderSubgroupMode::Controlled && + controlled.required_size == 64, + "supported controlled wave64 was not preferred over splitting"); + context.subgroup_size = 64; + context.subgroup_size_control_enabled = false; + cross_lane.wave_size = 32; + cross_lane.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eFragment, cross_lane) + .mode == ShaderSubgroupMode::Unsupported, + "inverse graphics mismatch was accepted as one guest wave"); + cross_lane_compute.wave_size = 32; + Check(ConfigureShaderSubgroup(ShaderSubgroupCapabilities{context}, + vk::ShaderStageFlagBits::eCompute, + cross_lane_compute) + .mode == ShaderSubgroupMode::Unsupported, + "inverse cross-lane compute mismatch was accepted"); } std::array ImageTestUserData(Prospero::ImageType type = Prospero::ImageType::kColor2D) { - std::array data {}; - for (uint32_t start = 0; start + 3u < data.size(); start += 4u) { - data[start] = 0x1000u + start * 0x100u; - data[start + 1u] = static_cast(Prospero::BufferFormat::k8UNorm) << 20u; - data[start + 2u] = UINT32_MAX; - data[start + 3u] = static_cast(type) << 28u; - } - return data; + std::array data{}; + for (uint32_t start = 0; start + 3u < data.size(); start += 4u) { + data[start] = 0x1000u + start * 0x100u; + data[start + 1u] = static_cast(Prospero::BufferFormat::k8UNorm) + << 20u; + data[start + 2u] = UINT32_MAX; + data[start + 3u] = static_cast(type) << 28u; + } + return data; } -void SetImageTestType(std::array* data, uint32_t srsrc, Prospero::ImageType type) { - const auto type_dword = srsrc * 4u + 3u; - Check(data != nullptr && type_dword < data->size(), "invalid image test descriptor source"); - (*data)[type_dword] = static_cast(type) << 28u; +void SetImageTestType(std::array *data, uint32_t srsrc, + Prospero::ImageType type) { + const auto type_dword = srsrc * 4u + 3u; + Check(data != nullptr && type_dword < data->size(), + "invalid image test descriptor source"); + (*data)[type_dword] = static_cast(type) << 28u; } -void SetImageTestFormat(std::array* data, uint32_t srsrc, +void SetImageTestFormat(std::array *data, uint32_t srsrc, Prospero::BufferFormat format) { - const auto format_dword = srsrc * 4u + 1u; - Check(data != nullptr && format_dword < data->size(), "invalid image test descriptor source"); - (*data)[format_dword] = static_cast(format) << 20u; + const auto format_dword = srsrc * 4u + 1u; + Check(data != nullptr && format_dword < data->size(), + "invalid image test descriptor source"); + (*data)[format_dword] = static_cast(format) << 20u; } -bool ReadZeroTestMemory(void*, uint64_t, uint32_t* value) { - if (value == nullptr) { - return false; - } - *value = 0; - return true; +bool ReadZeroTestMemory(void *, uint64_t, uint32_t *value) { + if (value == nullptr) { + return false; + } + *value = 0; + return true; } -constexpr uint32_t EncodeVop2(uint32_t opcode, uint32_t dst, uint32_t src0, uint32_t src1) { - return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | - (src0 & 0x1ffu); +constexpr uint32_t EncodeVop2(uint32_t opcode, uint32_t dst, uint32_t src0, + uint32_t src1) { + return ((opcode & 0x3fu) << 25u) | ((dst & 0xffu) << 17u) | + ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu); } constexpr uint32_t EncodeVop1(uint32_t opcode, uint32_t dst, uint32_t src0) { - return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) | (src0 & 0x1ffu); + return (0x3fu << 25u) | ((dst & 0xffu) << 17u) | ((opcode & 0xffu) << 9u) | + (src0 & 0x1ffu); } -constexpr uint32_t EncodeVop1Sdwa(uint32_t src0, uint32_t dst_sel = 6, uint32_t dst_u = 0, - uint32_t src0_sel = 6, uint32_t src0_sext = 0, - uint32_t src0_neg = 0, uint32_t src0_abs = 0, uint32_t s0 = 0) { - return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | - ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | - ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u); +constexpr uint32_t EncodeVop1Sdwa(uint32_t src0, uint32_t dst_sel = 6, + uint32_t dst_u = 0, uint32_t src0_sel = 6, + uint32_t src0_sext = 0, uint32_t src0_neg = 0, + uint32_t src0_abs = 0, uint32_t s0 = 0) { + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u); } -constexpr uint32_t EncodeVop1Dpp(uint32_t src0, uint32_t dpp_ctrl = 0, uint32_t row_mask = 0xf, +constexpr uint32_t EncodeVop1Dpp(uint32_t src0, uint32_t dpp_ctrl = 0, + uint32_t row_mask = 0xf, uint32_t bank_mask = 0xf) { - return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | ((bank_mask & 0xfu) << 24u) | - ((row_mask & 0xfu) << 28u); + return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | + ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); } -constexpr uint32_t EncodeVop2Sdwa(uint32_t src0, uint32_t dst_sel = 6, uint32_t dst_u = 0, - uint32_t src0_sel = 6, uint32_t src1_sel = 6, - uint32_t src0_sext = 0, uint32_t src1_sext = 0, - uint32_t src0_neg = 0, uint32_t src0_abs = 0, - uint32_t src1_neg = 0, uint32_t src1_abs = 0, uint32_t s0 = 0, +constexpr uint32_t EncodeVop2Sdwa(uint32_t src0, uint32_t dst_sel = 6, + uint32_t dst_u = 0, uint32_t src0_sel = 6, + uint32_t src1_sel = 6, uint32_t src0_sext = 0, + uint32_t src1_sext = 0, uint32_t src0_neg = 0, + uint32_t src0_abs = 0, uint32_t src1_neg = 0, + uint32_t src1_abs = 0, uint32_t s0 = 0, uint32_t s1 = 0) { - return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | - ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | - ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | - ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | ((src1_abs & 0x1u) << 29u) | - ((s1 & 0x1u) << 31u); + return (src0 & 0xffu) | ((dst_sel & 0x7u) << 8u) | ((dst_u & 0x3u) << 11u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | + ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); } -constexpr uint32_t EncodeVop2Dpp(uint32_t src0, uint32_t dpp_ctrl = 0, uint32_t row_mask = 0xf, - uint32_t bank_mask = 0xf, uint32_t src0_neg = 0, - uint32_t src0_abs = 0, uint32_t src1_neg = 0, - uint32_t src1_abs = 0) { - return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | ((src0_neg & 0x1u) << 20u) | - ((src0_abs & 0x1u) << 21u) | ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) | - ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); +constexpr uint32_t EncodeVop2Dpp(uint32_t src0, uint32_t dpp_ctrl = 0, + uint32_t row_mask = 0xf, + uint32_t bank_mask = 0xf, + uint32_t src0_neg = 0, uint32_t src0_abs = 0, + uint32_t src1_neg = 0, uint32_t src1_abs = 0) { + return (src0 & 0xffu) | ((dpp_ctrl & 0x1ffu) << 8u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((src1_neg & 0x1u) << 22u) | ((src1_abs & 0x1u) << 23u) | + ((bank_mask & 0xfu) << 24u) | ((row_mask & 0xfu) << 28u); } constexpr uint32_t EncodeVopc(uint32_t opcode, uint32_t src0, uint32_t src1) { - return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | (src0 & 0x1ffu); + return (0x3eu << 25u) | ((opcode & 0xffu) << 17u) | ((src1 & 0xffu) << 9u) | + (src0 & 0x1ffu); } -constexpr uint32_t EncodeVopcSdwa(uint32_t src0, uint32_t sdst = 0, uint32_t sd = 0, - uint32_t src0_sel = 6, uint32_t src1_sel = 6, - uint32_t src0_sext = 0, uint32_t src1_sext = 0, - uint32_t src0_neg = 0, uint32_t src0_abs = 0, - uint32_t src1_neg = 0, uint32_t src1_abs = 0, uint32_t s0 = 0, +constexpr uint32_t EncodeVopcSdwa(uint32_t src0, uint32_t sdst = 0, + uint32_t sd = 0, uint32_t src0_sel = 6, + uint32_t src1_sel = 6, uint32_t src0_sext = 0, + uint32_t src1_sext = 0, uint32_t src0_neg = 0, + uint32_t src0_abs = 0, uint32_t src1_neg = 0, + uint32_t src1_abs = 0, uint32_t s0 = 0, uint32_t s1 = 0) { - return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) | - ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | ((src0_neg & 0x1u) << 20u) | - ((src0_abs & 0x1u) << 21u) | ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | - ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | ((src1_abs & 0x1u) << 29u) | - ((s1 & 0x1u) << 31u); + return (src0 & 0xffu) | ((sdst & 0x7fu) << 8u) | ((sd & 0x1u) << 15u) | + ((src0_sel & 0x7u) << 16u) | ((src0_sext & 0x1u) << 19u) | + ((src0_neg & 0x1u) << 20u) | ((src0_abs & 0x1u) << 21u) | + ((s0 & 0x1u) << 23u) | ((src1_sel & 0x7u) << 24u) | + ((src1_sext & 0x1u) << 27u) | ((src1_neg & 0x1u) << 28u) | + ((src1_abs & 0x1u) << 29u) | ((s1 & 0x1u) << 31u); } -constexpr uint32_t EncodeVop3Word0(uint32_t opcode, uint32_t dst, uint32_t op_sel = 0, - uint32_t abs = 0, bool clamp = false) { - return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | (dst & 0xffu) | ((abs & 0x7u) << 8u) | - ((op_sel & 0xfu) << 11u) | (clamp ? (1u << 15u) : 0u); +constexpr uint32_t EncodeVop3Word0(uint32_t opcode, uint32_t dst, + uint32_t op_sel = 0, uint32_t abs = 0, + bool clamp = false) { + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | (dst & 0xffu) | + ((abs & 0x7u) << 8u) | ((op_sel & 0xfu) << 11u) | + (clamp ? (1u << 15u) : 0u); } -constexpr uint32_t EncodeVop3Word0Sdst(uint32_t opcode, uint32_t dst, uint32_t sdst) { - return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) | (dst & 0xffu); +constexpr uint32_t EncodeVop3Word0Sdst(uint32_t opcode, uint32_t dst, + uint32_t sdst) { + return (0x35u << 26u) | ((opcode & 0x3ffu) << 16u) | ((sdst & 0x7fu) << 8u) | + (dst & 0xffu); } -constexpr uint32_t EncodeVop3Word1(uint32_t src0, uint32_t src1, uint32_t src2) { - return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u); +constexpr uint32_t EncodeVop3Word1(uint32_t src0, uint32_t src1, + uint32_t src2) { + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u); } -constexpr uint32_t EncodeVop3pWord0(uint32_t opcode, uint32_t dst, uint32_t op_sel = 0, - uint32_t op_sel_hi = 0, uint32_t neg_hi = 0, - bool clamp = false) { - return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | (dst & 0xffu) | ((neg_hi & 0x7u) << 8u) | - ((op_sel & 0x7u) << 11u) | (((op_sel_hi >> 2u) & 0x1u) << 14u) | - (clamp ? (1u << 15u) : 0u); +constexpr uint32_t EncodeVop3pWord0(uint32_t opcode, uint32_t dst, + uint32_t op_sel = 0, uint32_t op_sel_hi = 0, + uint32_t neg_hi = 0, bool clamp = false) { + return (0x33u << 26u) | ((opcode & 0x7fu) << 16u) | (dst & 0xffu) | + ((neg_hi & 0x7u) << 8u) | ((op_sel & 0x7u) << 11u) | + (((op_sel_hi >> 2u) & 0x1u) << 14u) | (clamp ? (1u << 15u) : 0u); } constexpr uint32_t EncodeVop3pWord1(uint32_t src0, uint32_t src1, uint32_t src2, uint32_t op_sel_hi = 0, uint32_t neg = 0) { - return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | - ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u); + return (src0 & 0x1ffu) | ((src1 & 0x1ffu) << 9u) | ((src2 & 0x1ffu) << 18u) | + ((op_sel_hi & 0x3u) << 27u) | ((neg & 0x7u) << 29u); } constexpr uint32_t EncodeSmem0(uint32_t opcode, uint32_t dst, uint32_t sbase) { - return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) | (sbase & 0x3fu); + return (0x3du << 26u) | ((opcode & 0xffu) << 18u) | ((dst & 0x7fu) << 6u) | + (sbase & 0x3fu); } -constexpr uint32_t EncodeMubuf0(uint32_t opcode, uint32_t offset = 0, bool idxen = true, - bool glc = false) { - return (0x38u << 26u) | ((opcode & 0x7fu) << 18u) | (idxen ? (1u << 13u) : 0u) | - (glc ? (1u << 14u) : 0u) | (offset & 0xfffu); +constexpr uint32_t EncodeMubuf0(uint32_t opcode, uint32_t offset = 0, + bool idxen = true, bool glc = false) { + return (0x38u << 26u) | ((opcode & 0x7fu) << 18u) | + (idxen ? (1u << 13u) : 0u) | (glc ? (1u << 14u) : 0u) | + (offset & 0xfffu); } constexpr uint32_t EncodeMubuf1(uint32_t vdata, uint32_t srsrc, uint32_t vaddr, uint32_t soffset = 128) { - return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | - (vaddr & 0xffu); + return ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); } -constexpr uint32_t EncodeMtbuf0(uint32_t opcode, uint32_t dfmt, uint32_t nfmt, uint32_t offset = 0, - bool idxen = true, bool offen = false) { - return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) | - (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) | ((dfmt & 0xfu) << 19u) | - ((nfmt & 0x7u) << 23u); +constexpr uint32_t EncodeMtbuf0(uint32_t opcode, uint32_t dfmt, uint32_t nfmt, + uint32_t offset = 0, bool idxen = true, + bool offen = false) { + return (0x3au << 26u) | (offset & 0xfffu) | (offen ? (1u << 12u) : 0u) | + (idxen ? (1u << 13u) : 0u) | ((opcode & 0x7u) << 16u) | + ((dfmt & 0xfu) << 19u) | ((nfmt & 0x7u) << 23u); } -constexpr uint32_t EncodeMtbuf1(uint32_t opcode, uint32_t vdata, uint32_t srsrc, uint32_t vaddr, - uint32_t soffset = 128) { - return ((opcode >> 3u) & 1u) << 21u | ((soffset & 0xffu) << 24u) | ((srsrc & 0x1fu) << 16u) | - ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); +constexpr uint32_t EncodeMtbuf1(uint32_t opcode, uint32_t vdata, uint32_t srsrc, + uint32_t vaddr, uint32_t soffset = 128) { + return ((opcode >> 3u) & 1u) << 21u | ((soffset & 0xffu) << 24u) | + ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | (vaddr & 0xffu); } -constexpr uint32_t EncodeFlat0(uint32_t opcode, uint32_t seg, uint32_t offset = 0) { - return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) | ((seg & 0x3u) << 14u) | (offset & 0xfffu); +constexpr uint32_t EncodeFlat0(uint32_t opcode, uint32_t seg, + uint32_t offset = 0) { + return (0x37u << 26u) | ((opcode & 0x7fu) << 18u) | ((seg & 0x3u) << 14u) | + (offset & 0xfffu); } -constexpr uint32_t EncodeFlat1(uint32_t vdst, uint32_t saddr, uint32_t data, uint32_t addr) { - return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) | ((data & 0xffu) << 8u) | - (addr & 0xffu); +constexpr uint32_t EncodeFlat1(uint32_t vdst, uint32_t saddr, uint32_t data, + uint32_t addr) { + return ((vdst & 0xffu) << 24u) | ((saddr & 0x7fu) << 16u) | + ((data & 0xffu) << 8u) | (addr & 0xffu); } constexpr uint32_t EncodeDs0(uint32_t opcode, uint32_t offset = 0) { - return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (offset & 0xffffu); + return (0x36u << 26u) | ((opcode & 0xffu) << 18u) | (offset & 0xffffu); } -constexpr uint32_t EncodeDs1Ex(uint32_t vdst, uint32_t data1, uint32_t data0, uint32_t addr) { - return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) | ((data0 & 0xffu) << 8u) | - (addr & 0xffu); +constexpr uint32_t EncodeDs1Ex(uint32_t vdst, uint32_t data1, uint32_t data0, + uint32_t addr) { + return ((vdst & 0xffu) << 24u) | ((data1 & 0xffu) << 16u) | + ((data0 & 0xffu) << 8u) | (addr & 0xffu); } constexpr uint32_t EncodeDs1(uint32_t vdst, uint32_t data0, uint32_t addr) { - return EncodeDs1Ex(vdst, 0, data0, addr); + return EncodeDs1Ex(vdst, 0, data0, addr); } -constexpr uint32_t EncodeMimg0(uint32_t opcode, uint32_t dmask, bool glc = false, - uint32_t dim = 1) { - return (0x3cu << 26u) | ((opcode & 0x7fu) << 18u) | ((dmask & 0xfu) << 8u) | - ((dim & 0x7u) << 3u) | (glc ? (1u << 13u) : 0u) | ((opcode >> 7u) & 1u); +constexpr uint32_t EncodeMimg0(uint32_t opcode, uint32_t dmask, + bool glc = false, uint32_t dim = 1) { + return (0x3cu << 26u) | ((opcode & 0x7fu) << 18u) | ((dmask & 0xfu) << 8u) | + ((dim & 0x7u) << 3u) | (glc ? (1u << 13u) : 0u) | + ((opcode >> 7u) & 1u); } -constexpr uint32_t EncodeMimg1(uint32_t vdata, uint32_t srsrc, uint32_t ssamp, uint32_t vaddr, - bool a16 = false) { - return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | ((vdata & 0xffu) << 8u) | - (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u); +constexpr uint32_t EncodeMimg1(uint32_t vdata, uint32_t srsrc, uint32_t ssamp, + uint32_t vaddr, bool a16 = false) { + return ((ssamp & 0x1fu) << 21u) | ((srsrc & 0x1fu) << 16u) | + ((vdata & 0xffu) << 8u) | (vaddr & 0xffu) | (a16 ? (1u << 30u) : 0u); } -constexpr uint32_t EncodeVintrp(uint32_t opcode, uint32_t vdst, uint32_t attr, uint32_t chan, - uint32_t vsrc) { - return (0x32u << 26u) | ((opcode & 0x3u) << 16u) | ((vdst & 0xffu) << 18u) | - ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (vsrc & 0xffu); +constexpr uint32_t EncodeVintrp(uint32_t opcode, uint32_t vdst, uint32_t attr, + uint32_t chan, uint32_t vsrc) { + return (0x32u << 26u) | ((opcode & 0x3u) << 16u) | ((vdst & 0xffu) << 18u) | + ((attr & 0x3fu) << 10u) | ((chan & 0x3u) << 8u) | (vsrc & 0xffu); } -constexpr uint32_t EncodeExp0(uint32_t target, uint32_t en, bool done = true, bool compr = false, - bool vm = false) { - return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) | (compr ? (1u << 10u) : 0u) | - (done ? (1u << 11u) : 0u) | (vm ? (1u << 12u) : 0u); +constexpr uint32_t EncodeExp0(uint32_t target, uint32_t en, bool done = true, + bool compr = false, bool vm = false) { + return (0x3eu << 26u) | ((target & 0x3fu) << 4u) | (en & 0xfu) | + (compr ? (1u << 10u) : 0u) | (done ? (1u << 11u) : 0u) | + (vm ? (1u << 12u) : 0u); } -constexpr uint32_t EncodeExp1(uint32_t src0, uint32_t src1, uint32_t src2, uint32_t src3) { - return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) | - ((src3 & 0xffu) << 24u); +constexpr uint32_t EncodeExp1(uint32_t src0, uint32_t src1, uint32_t src2, + uint32_t src3) { + return (src0 & 0xffu) | ((src1 & 0xffu) << 8u) | ((src2 & 0xffu) << 16u) | + ((src3 & 0xffu) << 24u); } void TestNewShaderRecompilerSMovB32() { - const uint32_t shader[] = { - EncodeSMovB32(0, 129), // s_mov_b32 s0, 1 - EncodeSMovB32(1, 255), // s_mov_b32 s1, 0x12345678 - 0x12345678u, EncodeSMovB32(2, 1), // s_mov_b32 s2, s1 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 129), // s_mov_b32 s0, 1 + EncodeSMovB32(1, 255), // s_mov_b32 s1, 0x12345678 + 0x12345678u, + EncodeSMovB32(2, 1), // s_mov_b32 s2, s1 + EncodeVop1(0x01, 0, 0), + EncodeMubuf0(0x1c, 0, false), + EncodeMubuf1(0, 12, 0), + EncodeVop1(0x01, 1, 1), + EncodeMubuf0(0x1c, 4, false), + EncodeMubuf1(1, 12, 0), + EncodeVop1(0x01, 2, 2), + EncodeMubuf0(0x1c, 8, false), + EncodeMubuf1(2, 12, 0), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(!result.spirv.empty(), "new shader recompiler produced no SPIR-V"); - Check(result.spirv.front() == 0x07230203u, "new shader recompiler did not emit SPIR-V binary"); - Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s0, 1"), - "new decoder did not decode inline S_MOV_B32 operand"); - Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s1, 0x12345678"), - "new decoder did not decode literal S_MOV_B32 operand"); - Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s2, s1"), - "new decoder did not decode register S_MOV_B32 operand"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 s0, 0x00000001"), - "new IR did not lower inline S_MOV_B32"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 s1, 0x12345678"), - "new IR did not lower literal S_MOV_B32"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 s2, s1"), - "new IR did not lower register S_MOV_B32"); - Check(std::find(result.spirv.begin(), result.spirv.end(), 0x12345678u) != result.spirv.end(), - "new SPIR-V emitter did not encode the literal as a binary word"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(!result.spirv.empty(), "new shader recompiler produced no SPIR-V"); + Check(result.spirv.front() == 0x07230203u, + "new shader recompiler did not emit SPIR-V binary"); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s0, 1"), + "new decoder did not decode inline S_MOV_B32 operand"); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s1, 0x12345678"), + "new decoder did not decode literal S_MOV_B32 operand"); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b32 s2, s1"), + "new decoder did not decode register S_MOV_B32 operand"); + Check(Common::ContainsStr(result.ir_dump, "StoreBufferU32") && + Common::ContainsStr(result.ir_dump, "0x00000001") && + Common::ContainsStr(result.ir_dump, "0x12345678"), + "typed SSA did not preserve live S_MOV_B32 values"); + Check(!Common::ContainsStr(result.ir_dump, "SetScalarRegister") && + !Common::ContainsStr(result.ir_dump, "GetScalarRegister"), + "typed SSA retained register-state pseudo operations"); + Check(std::find(result.spirv.begin(), result.spirv.end(), 0x12345678u) != + result.spirv.end(), + "new SPIR-V emitter did not encode the literal as a binary word"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerSoppMarkers() { - const uint32_t shader[] = { - EncodeSopp(0x00, 3), // s_nop 3 - EncodeSopp(0x0c, 0), // s_waitcnt 0 - EncodeSopp(0x10, 0x0f), // s_sendmsg 15 - EncodeSopp(0x16, 0x2a), // s_ttracedata 42 - EncodeSopp(0x20, 1), // s_inst_prefetch 1 - EncodeSopp(0x0a, 0), // s_barrier - EncodeSopp(0x01, 0), // s_endpgm - }; + const uint32_t shader[] = { + EncodeSopp(0x00, 3), // s_nop 3 + EncodeSopp(0x0c, 0), // s_waitcnt 0 + EncodeSopp(0x10, 0x0f), // s_sendmsg 15 + EncodeSopp(0x16, 0x2a), // s_ttracedata 42 + EncodeSopp(0x20, 1), // s_inst_prefetch 1 + EncodeSopp(0x0a, 0), // s_barrier + EncodeSopp(0x01, 0), // s_endpgm + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_nop 0x00000003"), - "new decoder did not decode SOPP s_nop"); - Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0x00000000"), - "new decoder did not decode SOPP s_waitcnt"); - Check(Common::ContainsStr(result.decoded_dump, "s_sendmsg 0x0000000f"), - "new decoder did not decode SOPP s_sendmsg"); - Check(Common::ContainsStr(result.decoded_dump, "s_ttracedata 0x0000002a"), - "new decoder did not decode SOPP s_ttracedata"); - Check(Common::ContainsStr(result.decoded_dump, "s_inst_prefetch 0x00000001"), - "new decoder did not decode SOPP s_inst_prefetch"); - Check(Common::ContainsStr(result.decoded_dump, "s_barrier"), - "new decoder did not decode SOPP s_barrier"); - Check(Common::ContainsStr(result.ir_dump, "ControlNop null, 0x00000003"), - "SOPP s_nop did not lower to an IR marker"); - Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), - "SOPP s_waitcnt did not lower to an IR marker"); - Check(Common::ContainsStr(result.ir_dump, "Sendmsg null, 0x0000000f"), - "SOPP s_sendmsg did not lower to an IR marker"); - Check(Common::ContainsStr(result.ir_dump, "TtraceData null, 0x0000002a"), - "SOPP s_ttracedata did not lower to an IR marker"); - Check(Common::ContainsStr(result.ir_dump, "InstPrefetch null, 0x00000001"), - "SOPP s_inst_prefetch did not lower to an IR marker"); - Check(Common::ContainsStr(result.ir_dump, "Barrier null"), - "SOPP s_barrier did not lower to an IR marker"); - Check(SpirvContainsOpcode(result.spirv, 224), - "SPIR-V binary does not contain OpControlBarrier"); - Check(std::find(result.spirv.begin(), result.spirv.end(), 264u) != result.spirv.end(), - "SPIR-V barrier does not use workgroup acquire-release memory semantics"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_nop 0x00000003"), + "new decoder did not decode SOPP s_nop"); + Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0x00000000"), + "new decoder did not decode SOPP s_waitcnt"); + Check(Common::ContainsStr(result.decoded_dump, "s_sendmsg 0x0000000f"), + "new decoder did not decode SOPP s_sendmsg"); + Check(Common::ContainsStr(result.decoded_dump, "s_ttracedata 0x0000002a"), + "new decoder did not decode SOPP s_ttracedata"); + Check(Common::ContainsStr(result.decoded_dump, "s_inst_prefetch 0x00000001"), + "new decoder did not decode SOPP s_inst_prefetch"); + Check(Common::ContainsStr(result.decoded_dump, "s_barrier"), + "new decoder did not decode SOPP s_barrier"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop null, 0x00000003"), + "SOPP s_nop did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), + "SOPP s_waitcnt did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Sendmsg null, 0x0000000f"), + "SOPP s_sendmsg did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "TtraceData null, 0x0000002a"), + "SOPP s_ttracedata did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "InstPrefetch null, 0x00000001"), + "SOPP s_inst_prefetch did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Barrier null"), + "SOPP s_barrier did not lower to an IR marker"); + Check(SpirvContainsOpcode(result.spirv, 224), + "SPIR-V binary does not contain OpControlBarrier"); + Check( + std::find(result.spirv.begin(), result.spirv.end(), 264u) != + result.spirv.end(), + "SPIR-V barrier does not use workgroup acquire-release memory semantics"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerSopkWaitcntMarkers() { - const uint32_t shader[] = { - EncodeSopk(0x17, 125, 0xffff), // s_waitcnt_vscnt null, 0xffff - EncodeSopk(0x18, 125, 0), // s_waitcnt_vmcnt null, 0 - EncodeSopk(0x19, 125, 0), // s_waitcnt_expcnt null, 0 - EncodeSopk(0x1a, 125, 0), // s_waitcnt_lgkmcnt null, 0 - EncodeSopp(0x01, 0), // s_endpgm - }; + const uint32_t shader[] = { + EncodeSopk(0x17, 125, 0xffff), // s_waitcnt_vscnt null, 0xffff + EncodeSopk(0x18, 125, 0), // s_waitcnt_vmcnt null, 0 + EncodeSopk(0x19, 125, 0), // s_waitcnt_expcnt null, 0 + EncodeSopk(0x1a, 125, 0), // s_waitcnt_lgkmcnt null, 0 + EncodeSopp(0x01, 0), // s_endpgm + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0"), - "new decoder did not decode SOPK waitcnt marker"); - Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 65535"), - "SOPK waitcnt marker immediate was not kept unsigned"); - Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), - "SOPK waitcnt did not lower to an IR marker"); - Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x0000ffff"), - "SOPK waitcnt marker immediate was not lowered as 16-bit unsigned"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0"), + "new decoder did not decode SOPK waitcnt marker"); + Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 65535"), + "SOPK waitcnt marker immediate was not kept unsigned"); + Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x00000000"), + "SOPK waitcnt did not lower to an IR marker"); + Check(Common::ContainsStr(result.ir_dump, "Waitcnt null, 0x0000ffff"), + "SOPK waitcnt marker immediate was not lowered as 16-bit unsigned"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerRdna2ScalarOpcodes() { - const uint32_t shader[] = { - EncodeSMovB32(2, 135), // s2 = 7 - EncodeSMovB32(106, 144), // vcc_lo = 16 - EncodeSop1(0x1d, 106, 128), // s_bitset1_b32 vcc_lo, 0 - EncodeSopk(0x13, 106, 0x1019), // s_setreg_b32 vcc_lo, 0x1019 - EncodeSop2(0x02, 106, 2, 239), // s_add_i32 vcc_lo, s2, pops_exiting_wave_id - EncodeSopp(0x0e, 0), // s_sleep 0 - EncodeSop2(0x02, 106, 239, 2), // s_add_i32 vcc_lo, pops_exiting_wave_id, s2 - EncodeSopp(0x01, 0), // s_endpgm - }; + const uint32_t shader[] = { + EncodeSMovB32(2, 135), // s2 = 7 + EncodeSMovB32(106, 144), // vcc_lo = 16 + EncodeSop1(0x1d, 106, 128), // s_bitset1_b32 vcc_lo, 0 + EncodeSopk(0x13, 106, 0x1019), // s_setreg_b32 vcc_lo, 0x1019 + EncodeSop2(0x02, 106, 2, + 239), // s_add_i32 vcc_lo, s2, pops_exiting_wave_id + EncodeSopp(0x0e, 0), // s_sleep 0 + EncodeSop2(0x02, 106, 239, + 2), // s_add_i32 vcc_lo, pops_exiting_wave_id, s2 + EncodeSopp(0x01, 0), // s_endpgm + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_bitset1_b32 vcc_lo, 0"), - "new decoder did not decode RDNA2 S_BITSET1_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_setreg_b32 vcc_lo, 0x00001019"), - "new decoder did not decode S_SETREG_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_add_i32 vcc_lo, s2, pops_exiting_wave_id"), - "new decoder did not decode pops_exiting_wave_id as RHS scalar source"); - Check(Common::ContainsStr(result.decoded_dump, "s_add_i32 vcc_lo, pops_exiting_wave_id, s2"), - "new decoder did not decode pops_exiting_wave_id as LHS scalar source"); - Check(Common::ContainsStr(result.decoded_dump, "s_sleep 0x00000000"), - "new decoder did not decode S_SLEEP"); - Check(Common::ContainsStr(result.ir_dump, "BitSetU32 vcc_lo, vcc_lo, 0x00000000"), - "S_BITSET1_B32 did not lower to bit-set IR using the destination as input"); - Check(Common::ContainsStr(result.ir_dump, "ScalarSignedAddOverflowI32 vcc_lo, s2, 0x00000000"), - "pops_exiting_wave_id RHS did not lower to a deterministic zero value"); - Check(Common::ContainsStr(result.ir_dump, "ScalarSignedAddOverflowI32 vcc_lo, 0x00000000, s2"), - "pops_exiting_wave_id LHS did not lower to a deterministic zero value"); - Check(Common::ContainsStr(result.ir_dump, "ControlNop null, vcc_lo"), - "S_SETREG_B32 did not lower to an explicit control marker"); - Check(Common::ContainsStr(result.ir_dump, "ControlNop null, 0x00000000"), - "S_SLEEP did not lower to an explicit control marker"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical for S_BITSET1_B32"); - Check(SpirvContainsOpcode(result.spirv, 197), - "SPIR-V binary does not contain OpBitwiseOr for S_BITSET1_B32"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_bitset1_b32 vcc_lo, 0"), + "new decoder did not decode RDNA2 S_BITSET1_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "s_setreg_b32 vcc_lo, 0x00001019"), + "new decoder did not decode S_SETREG_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "s_add_i32 vcc_lo, s2, pops_exiting_wave_id"), + "new decoder did not decode pops_exiting_wave_id as RHS scalar source"); + Check(Common::ContainsStr(result.decoded_dump, + "s_add_i32 vcc_lo, pops_exiting_wave_id, s2"), + "new decoder did not decode pops_exiting_wave_id as LHS scalar source"); + Check(Common::ContainsStr(result.decoded_dump, "s_sleep 0x00000000"), + "new decoder did not decode S_SLEEP"); + Check(Common::ContainsStr(result.ir_dump, + "BitSetU32 vcc_lo, vcc_lo, 0x00000000"), + "S_BITSET1_B32 did not lower to bit-set IR using the destination as " + "input"); + Check( + Common::ContainsStr(result.ir_dump, + "ScalarSignedAddOverflowI32 vcc_lo, s2, 0x00000000"), + "pops_exiting_wave_id RHS did not lower to a deterministic zero value"); + Check( + Common::ContainsStr(result.ir_dump, + "ScalarSignedAddOverflowI32 vcc_lo, 0x00000000, s2"), + "pops_exiting_wave_id LHS did not lower to a deterministic zero value"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop null, vcc_lo"), + "S_SETREG_B32 did not lower to an explicit control marker"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop null, 0x00000000"), + "S_SLEEP did not lower to an explicit control marker"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for S_BITSET1_B32"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for S_BITSET1_B32"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerScalarVectorAlu() { - const uint32_t shader[] = { - EncodeSMovB32(0, 129), // s0 = 1 - EncodeSMovB32(1, 130), // s1 = 2 - EncodeSop2(0x00, 2, 0, 1), // s_add_u32 s2, s0, s1 - EncodeSop2(0x01, 3, 2, 129), // s_sub_u32 s3, s2, 1 - EncodeSop2(0x0e, 4, 2, 3), // s_and_b32 s4, s2, s3 - EncodeSop2(0x10, 5, 2, 3), // s_or_b32 s5, s2, s3 - EncodeSop2(0x12, 6, 2, 3), // s_xor_b32 s6, s2, s3 - EncodeSop2(0x1e, 7, 2, 129), // s_lshl_b32 s7, s2, 1 - EncodeSop2(0x20, 8, 7, 129), // s_lshr_b32 s8, s7, 1 - EncodeSop2(0x2e, 9, 2, 1), // s_lshl1_add_u32 s9, s2, s1 - EncodeSop2(0x2f, 10, 9, 1), // s_lshl2_add_u32 s10, s9, s1 - EncodeSop2(0x30, 11, 10, 1), // s_lshl3_add_u32 s11, s10, s1 - EncodeSop2(0x31, 12, 11, 1), // s_lshl4_add_u32 s12, s11, s1 - EncodeSop2(0x35, 13, 12, 1), // s_mul_hi_u32 s13, s12, s1 - EncodeSopc(0x08, 8, 1), // s_cmp_gt_u32 s8, s1 - EncodeSop2(0x04, 14, 13, 129), // s_addc_u32 s14, s13, 1 - EncodeVop2(0x03, 1, 242, 0), // v_add_f32 v1, 1.0, v0 - EncodeVop2(0x08, 2, 1 + 256, 1), // v_mul_f32 v2, v1, v1 - EncodeVop2(0x25, 3, 1 + 256, 2), // v_add_nc_u32 v3, v1, v2 - EncodeVop2(0x1b, 4, 3 + 256, 2), // v_and_b32 v4, v3, v2 - EncodeVop2(0x0b, 6, 3 + 256, 4), // v_mul_u32_u24 v6, v3, v4 - EncodeVop2(0x09, 8, 4 + 256, 6), // v_mul_i32_i24 v8, v4, v6 - EncodeVop2(0x22, 7, 4 + 256, 6), // v_bcnt_u32_b32 v7, v4, v6 - EncodeVopc(0xc4, 3 + 256, 2), // v_cmp_gt_u32 v3, v2 - EncodeVop2(0x01, 5, 1 + 256, 2), // v_cndmask_b32 v5, v1, v2 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 129), // s0 = 1 + EncodeSMovB32(1, 130), // s1 = 2 + EncodeSop2(0x00, 2, 0, 1), // s_add_u32 s2, s0, s1 + EncodeSop2(0x01, 3, 2, 129), // s_sub_u32 s3, s2, 1 + EncodeSop2(0x0e, 4, 2, 3), // s_and_b32 s4, s2, s3 + EncodeSop2(0x10, 5, 2, 3), // s_or_b32 s5, s2, s3 + EncodeSop2(0x12, 6, 2, 3), // s_xor_b32 s6, s2, s3 + EncodeSop2(0x1e, 7, 2, 129), // s_lshl_b32 s7, s2, 1 + EncodeSop2(0x20, 8, 7, 129), // s_lshr_b32 s8, s7, 1 + EncodeSop2(0x2e, 9, 2, 1), // s_lshl1_add_u32 s9, s2, s1 + EncodeSop2(0x2f, 10, 9, 1), // s_lshl2_add_u32 s10, s9, s1 + EncodeSop2(0x30, 11, 10, 1), // s_lshl3_add_u32 s11, s10, s1 + EncodeSop2(0x31, 12, 11, 1), // s_lshl4_add_u32 s12, s11, s1 + EncodeSop2(0x35, 13, 12, 1), // s_mul_hi_u32 s13, s12, s1 + EncodeSopc(0x08, 8, 1), // s_cmp_gt_u32 s8, s1 + EncodeSop2(0x04, 14, 13, 129), // s_addc_u32 s14, s13, 1 + EncodeVop2(0x03, 1, 242, 0), // v_add_f32 v1, 1.0, v0 + EncodeVop2(0x08, 2, 1 + 256, 1), // v_mul_f32 v2, v1, v1 + EncodeVop2(0x25, 3, 1 + 256, 2), // v_add_nc_u32 v3, v1, v2 + EncodeVop2(0x1b, 4, 3 + 256, 2), // v_and_b32 v4, v3, v2 + EncodeVop2(0x0b, 6, 3 + 256, 4), // v_mul_u32_u24 v6, v3, v4 + EncodeVop2(0x09, 8, 4 + 256, 6), // v_mul_i32_i24 v8, v4, v6 + EncodeVop2(0x22, 7, 4 + 256, 6), // v_bcnt_u32_b32 v7, v4, v6 + EncodeVopc(0xc4, 3 + 256, 2), // v_cmp_gt_u32 v3, v2 + EncodeVop2(0x01, 5, 1 + 256, 2), // v_cndmask_b32 v5, v1, v2 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_add_u32 s2, s0, s1"), - "new decoder did not decode SOP2 add"); - Check(Common::ContainsStr(result.decoded_dump, "s_addc_u32 s14, s13, 1"), - "new decoder did not decode old-backed S_ADD_C_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32 s8, s1"), - "new decoder did not decode SOPC compare"); - Check(Common::ContainsStr(result.decoded_dump, "s_lshl1_add_u32 s9, s2, s1"), - "new decoder did not decode old-backed S_LSHL1_ADD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_lshl2_add_u32 s10, s9, s1"), - "new decoder did not decode old-backed S_LSHL2_ADD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_lshl3_add_u32 s11, s10, s1"), - "new decoder did not decode old-backed S_LSHL3_ADD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_lshl4_add_u32 s12, s11, s1"), - "new decoder did not decode old-backed S_LSHL4_ADD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_mul_hi_u32 s13, s12, s1"), - "new decoder did not decode old-backed S_MUL_HI_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v1"), - "new decoder did not decode VOP2 float add"); - Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v5"), - "new decoder did not decode VOP2 conditional mask select"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_u32_u24 v6"), - "new decoder did not decode VOP2 24-bit multiply"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_i32_i24 v8"), - "new decoder did not decode VOP2 signed 24-bit multiply"); - Check(Common::ContainsStr(result.decoded_dump, "v_bcnt_u32_b32 v7"), - "new decoder did not decode old-backed V_BCNT_U32_B32"); - Check(Common::ContainsStr(result.ir_dump, "ScalarAddCarryU32 s2, s0, s1, 0x00000000"), - "SOP2 add did not lower to scalar carry-writing IR"); - Check(Common::ContainsStr(result.ir_dump, "ScalarAddCarryU32 s14, s13, 0x00000001, scc"), - "S_ADD_C_U32 did not lower to scalar carry IR"); - Check(Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s9, s2, 0x00000001, s1"), - "S_LSHL1_ADD_U32 did not lower through carry-writing shift-left-add IR"); - Check(Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s10, s9, 0x00000002, s1"), - "S_LSHL2_ADD_U32 did not lower through carry-writing shift-left-add IR"); - Check( - Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s11, s10, 0x00000003, s1"), - "S_LSHL3_ADD_U32 did not lower through carry-writing shift-left-add IR"); - Check( - Common::ContainsStr(result.ir_dump, "ScalarShiftLeftAddCarryU32 s12, s11, 0x00000004, s1"), - "S_LSHL4_ADD_U32 did not lower through carry-writing shift-left-add IR"); - Check(Common::ContainsStr(result.ir_dump, "UMulHighU32 s13, s12, s1"), - "S_MUL_HI_U32 did not lower to unsigned high-multiply IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareGtU32"), "SOPC compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FAddF32 v1"), "VOP2 float add did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU32 v4"), - "VOP2 bitwise op did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "SelectMaskU32 v5, vcc_lo, v2, v1"), - "VOP2 conditional mask did not lower through lane-mask select IR"); - Check(Common::ContainsStr(result.ir_dump, "UMulU24U32 v6"), - "VOP2 24-bit multiply did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMulI24U32 v8"), - "VOP2 signed 24-bit multiply did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitCountAddU32 v7, v4, v6"), - "V_BCNT_U32_B32 did not lower to bit-count-add IR"); - Check(SpirvContainsOpcode(result.spirv, 128), "SPIR-V binary does not contain OpIAdd"); - Check(SpirvContainsOpcode(result.spirv, 149), "SPIR-V binary does not contain OpIAddCarry"); - Check(SpirvContainsOpcode(result.spirv, 129), "SPIR-V binary does not contain OpFAdd"); - Check(SpirvContainsOpcode(result.spirv, 132), "SPIR-V binary does not contain OpIMul"); - Check(SpirvContainsOpcode(result.spirv, 151), "SPIR-V binary does not contain OpUMulExtended"); - Check(SpirvContainsOpcode(result.spirv, 205), "SPIR-V binary does not contain OpBitCount"); - Check(SpirvContainsOpcode(result.spirv, 169), "SPIR-V binary does not contain OpSelect"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_add_u32 s2, s0, s1"), + "new decoder did not decode SOP2 add"); + Check(Common::ContainsStr(result.decoded_dump, "s_addc_u32 s14, s13, 1"), + "new decoder did not decode old-backed S_ADD_C_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32 s8, s1"), + "new decoder did not decode SOPC compare"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl1_add_u32 s9, s2, s1"), + "new decoder did not decode old-backed S_LSHL1_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl2_add_u32 s10, s9, s1"), + "new decoder did not decode old-backed S_LSHL2_ADD_U32"); + Check( + Common::ContainsStr(result.decoded_dump, "s_lshl3_add_u32 s11, s10, s1"), + "new decoder did not decode old-backed S_LSHL3_ADD_U32"); + Check( + Common::ContainsStr(result.decoded_dump, "s_lshl4_add_u32 s12, s11, s1"), + "new decoder did not decode old-backed S_LSHL4_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_mul_hi_u32 s13, s12, s1"), + "new decoder did not decode old-backed S_MUL_HI_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v1"), + "new decoder did not decode VOP2 float add"); + Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v5"), + "new decoder did not decode VOP2 conditional mask select"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_u32_u24 v6"), + "new decoder did not decode VOP2 24-bit multiply"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_i32_i24 v8"), + "new decoder did not decode VOP2 signed 24-bit multiply"); + Check(Common::ContainsStr(result.decoded_dump, "v_bcnt_u32_b32 v7"), + "new decoder did not decode old-backed V_BCNT_U32_B32"); + Check(Common::ContainsStr(result.ir_dump, + "ScalarAddCarryU32 s2, s0, s1, 0x00000000"), + "SOP2 add did not lower to scalar carry-writing IR"); + Check(Common::ContainsStr(result.ir_dump, + "ScalarAddCarryU32 s14, s13, 0x00000001, scc"), + "S_ADD_C_U32 did not lower to scalar carry IR"); + Check( + Common::ContainsStr(result.ir_dump, + "ScalarShiftLeftAddCarryU32 s9, s2, 0x00000001, s1"), + "S_LSHL1_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check( + Common::ContainsStr(result.ir_dump, + "ScalarShiftLeftAddCarryU32 s10, s9, 0x00000002, s1"), + "S_LSHL2_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check( + Common::ContainsStr( + result.ir_dump, + "ScalarShiftLeftAddCarryU32 s11, s10, 0x00000003, s1"), + "S_LSHL3_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check( + Common::ContainsStr( + result.ir_dump, + "ScalarShiftLeftAddCarryU32 s12, s11, 0x00000004, s1"), + "S_LSHL4_ADD_U32 did not lower through carry-writing shift-left-add IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulHighU32 s13, s12, s1"), + "S_MUL_HI_U32 did not lower to unsigned high-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareGtU32"), + "SOPC compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FAddF32 v1"), + "VOP2 float add did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU32 v4"), + "VOP2 bitwise op did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SelectMaskU32 v5, vcc_lo, v2, v1"), + "VOP2 conditional mask did not lower through lane-mask select IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulU24U32 v6"), + "VOP2 24-bit multiply did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMulI24U32 v8"), + "VOP2 signed 24-bit multiply did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCountAddU32 v7, v4, v6"), + "V_BCNT_U32_B32 did not lower to bit-count-add IR"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd"); + Check(SpirvContainsOpcode(result.spirv, 149), + "SPIR-V binary does not contain OpIAddCarry"); + Check(SpirvContainsOpcode(result.spirv, 129), + "SPIR-V binary does not contain OpFAdd"); + Check(SpirvContainsOpcode(result.spirv, 132), + "SPIR-V binary does not contain OpIMul"); + Check(SpirvContainsOpcode(result.spirv, 151), + "SPIR-V binary does not contain OpUMulExtended"); + Check(SpirvContainsOpcode(result.spirv, 205), + "SPIR-V binary does not contain OpBitCount"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() { - // Lane-read instructions have scalar results, but their VOP3A destination is encoded in - // VDST [7:0], not the VOP3B SDST field [14:8]. - const uint32_t shader[] = { - EncodeVop3Word0(0x182, 25), EncodeVop3Word1(5 + 256, 0, 0), // v_readfirstlane_b32 s25, v5 - EncodeVop3Word0(0x360, 26), EncodeVop3Word1(5 + 256, 130, 0), // v_readlane_b32 s26, v5, 2 - EncodeSopp(0x01, 0), - }; + // Lane-read instructions have scalar results, but their VOP3A destination is + // encoded in VDST [7:0], not the VOP3B SDST field [14:8]. + const uint32_t shader[] = { + EncodeVop3Word0(0x182, 25), + EncodeVop3Word1(5 + 256, 0, 0), // v_readfirstlane_b32 s25, v5 + EncodeVop3Word0(0x360, 26), + EncodeVop3Word1(5 + 256, 130, 0), // v_readlane_b32 s26, v5, 2 + EncodeSopp(0x01, 0), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), - "VOP3 V_READFIRSTLANE_B32 destination was not decoded from VDST"); - Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), - "VOP3 V_READLANE_B32 destination was not decoded from VDST"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), + "VOP3 V_READFIRSTLANE_B32 destination was not decoded from VDST"); + Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), + "VOP3 V_READLANE_B32 destination was not decoded from VDST"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerMoreAluFamilies() { - const uint32_t shader[] = { - EncodeSopk(0x00, 9, 7), // s_movk_i32 s9, 7 - EncodeSopk(0x0b, 9, 3), // s_cmp_gt_u32 s9, 3 - EncodeVop1(0x00, 0, 0), // v_nop - EncodeVop1(0x01, 5, 9), // v_mov_b32 v5, s9 - EncodeVop1(0x01, 103, 249), - EncodeVop1Sdwa(5, 6, 0, 4), // v_mov_b32 v103, v5 word0 - EncodeVop1(0x06, 104, 249), - EncodeVop1Sdwa(103, 6, 0, 4), // v_cvt_f32_u32 v104, v103 word0 - EncodeVop1(0x0a, 105, 249), - EncodeVop1Sdwa(6, 5, 2, 6), // v_cvt_f16_f32 v105.hi, v6 - EncodeVop1(0x01, 106, 250), - EncodeVop1Dpp(5), // v_mov_b32 v106, v5 dpp - EncodeVop1(0x02, 24, 5 + 256), // v_readfirstlane_b32 s24, v5 - EncodeVop1(0x06, 6, 5 + 256), // v_cvt_f32_u32 v6, v5 - EncodeVop1(0x07, 7, 6 + 256), // v_cvt_u32_f32 v7, v6 - EncodeVop1(0x05, 9, 5 + 256), // v_cvt_f32_i32 v9, v5 - EncodeVop1(0x08, 10, 6 + 256), // v_cvt_i32_f32 v10, v6 - EncodeVop1(0x0a, 99, 6 + 256), // v_cvt_f16_f32 v99, v6 - EncodeVop1(0x0b, 100, 99 + 256), // v_cvt_f32_f16 v100, v99 - 0x7e1016f9u, - 0x00250602u, // v_cvt_f32_f16 v8, abs(v2.hi) - EncodeVop1(0x0d, 64, 6 + 256), // v_cvt_flr_i32_f32 v64, v6 - EncodeVop1(0x0e, 81, 5 + 256), // v_cvt_off_f32_i4 v81, v5 - EncodeVop1(0x11, 65, 5 + 256), // v_cvt_f32_ubyte0 v65, v5 - EncodeVop1(0x12, 66, 5 + 256), // v_cvt_f32_ubyte1 v66, v5 - EncodeVop1(0x13, 67, 5 + 256), // v_cvt_f32_ubyte2 v67, v5 - EncodeVop1(0x14, 68, 5 + 256), // v_cvt_f32_ubyte3 v68, v5 - EncodeVop1(0x2a, 11, 6 + 256), // v_rcp_f32 v11, v6 - EncodeVop1(0x20, 12, 6 + 256), // v_fract_f32 v12, v6 - EncodeVop1(0x21, 13, 6 + 256), // v_trunc_f32 v13, v6 - EncodeVop1(0x22, 14, 6 + 256), // v_ceil_f32 v14, v6 - EncodeVop1(0x23, 15, 6 + 256), // v_rndne_f32 v15, v6 - EncodeVop1(0x24, 16, 6 + 256), // v_floor_f32 v16, v6 - EncodeVop1(0x25, 17, 6 + 256), // v_exp_f32 v17, v6 - EncodeVop1(0x27, 18, 6 + 256), // v_log_f32 v18, v6 - EncodeVop1(0x2e, 19, 6 + 256), // v_rsq_f32 v19, v6 - EncodeVop1(0x33, 20, 6 + 256), // v_sqrt_f32 v20, v6 - EncodeVop1(0x35, 21, 6 + 256), // v_sin_f32 v21, v6 - EncodeVop1(0x36, 22, 6 + 256), // v_cos_f32 v22, v6 - EncodeVop1(0x37, 27, 5 + 256), // v_not_b32 v27, v5 - EncodeVop1(0x38, 28, 5 + 256), // v_bfrev_b32 v28, v5 - EncodeVop1(0x39, 31, 5 + 256), // v_ffbh_u32 v31, v5 - EncodeVop1(0x3a, 32, 5 + 256), // v_ffbl_b32 v32, v5 - 0x7e6e870cu, // v_movrels_b32 v55, v12 - EncodeVop2(0x1e, 83, 5 + 256, 6), // v_xnor_b32 v83, v5, v6 - EncodeVop2(0x23, 84, 5 + 256, 6), // v_mbcnt_lo_u32_b32 v84, v5, v6 - EncodeVop2(0x24, 85, 5 + 256, 6), // v_mbcnt_hi_u32_b32 v85, v5, v6 - EncodeVop2(0x1f, 89, 6 + 256, 6), // v_mac_f32 v89, v6, v6 - EncodeVop2(0x20, 90, 6 + 256, 5), - 0x3f800000u, // v_madmk_f32 v90, v6, 1.0, v5 - EncodeVop2(0x21, 91, 6 + 256, 5), - 0x40000000u, // v_madak_f32 v91, v6, v5, 2.0 - EncodeVop2(0x2b, 92, 6 + 256, 6), // v_mac_f32 v92, v6, v6 - EncodeVop2(0x2c, 93, 6 + 256, 5), - 0x3f000000u, // v_madmk_f32 v93, v6, 0.5, v5 - EncodeVop2(0x2d, 94, 6 + 256, 5), - 0x40400000u, // v_madak_f32 v94, v6, v5, 3.0 - EncodeVop2(0x2f, 95, 6 + 256, 6), // v_cvt_pkrtz_f16_f32 v95, v6, v6 - EncodeVop2(0x02, 102, 95 + 256, 95), // v_dot2c_f32_f16 v102, v95, v95 - EncodeVop2(0x28, 97, 5 + 256, 6), // v_addc_u32 v97, vcc, v5, v6, vcc - EncodeVop2(0x25, 123, 249, 6), - EncodeVop2Sdwa(5, 6, 0, 4, 6), - EncodeVop2(0x1b, 124, 249, 6), - EncodeVop2Sdwa(5, 6, 0, 4, 5), - 0x025e6af9u, - 0x16060635u, // v_cndmask_b32 v47, v53, -v53 - 0x020e02f9u, - 0x06040600u, // v_cndmask_b32 v7, v0.lo, v1; full-width destination - 0x100490f9u, - 0x86860600u, // v_mul_f32 v2, s0, s72 (SDWA full) - EncodeVop2(0x35, 9, 249, 1), - EncodeVop2Sdwa(0, 4, 2, 4, 4), - 0x660000f9u, - EncodeVop2Sdwa(0, 4, 2, 4, 4), - EncodeVop2(0x03, 125, 250, 6), - EncodeVop2Dpp(5), - EncodeVop3Word0(0x103, 35), - EncodeVop3Word1(6 + 256, 6 + 256, 0), - EncodeVop3Word0(0x10b, 36), - EncodeVop3Word1(5 + 256, 5 + 256, 0), - EncodeVop3Word0(0x11e, 86), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x11f, 96), - EncodeVop3Word1(6 + 256, 6 + 256, 0), - EncodeVop3Word0Sdst(0x128, 98, 30), - EncodeVop3Word1(5 + 256, 6 + 256, 106), - EncodeVop3Word0(0x123, 87), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x124, 88), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x180, 0), - EncodeVop3Word1(0, 0, 0), - EncodeVop3Word0(0x181, 23), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x182, 25), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x185, 24), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x18a, 101), - EncodeVop3Word1(6 + 256, 0, 0), - EncodeVop3Word0(0x18d, 74), - EncodeVop3Word1(6 + 256, 0, 0), - EncodeVop3Word0(0x18e, 82), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x1a0, 25), - EncodeVop3Word1(6 + 256, 0, 0), - EncodeVop3Word0(0x1aa, 26), - EncodeVop3Word1(6 + 256, 0, 0), - EncodeVop3Word0(0x1b7, 29), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x1b8, 30), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x1b9, 33), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVop3Word0(0x1ba, 34), - EncodeVop3Word1(5 + 256, 0, 0), - EncodeVopc(0x04, 6 + 256, 6), // v_cmp_gt_f32 v6, v6 - EncodeVopc(0xc4, 7 + 256, 5), // v_cmp_gt_u32 v7, v5 - EncodeVopc(0x14, 6 + 256, 6), // v_cmpx_gt_f32 v6, v6 - EncodeVopc(0xd4, 7 + 256, 5), // v_cmpx_gt_u32 v7, v5 - EncodeVopc(0x00, 6 + 256, 6), // v_cmp_f_f32 v6, v6 - EncodeVopc(0x0f, 6 + 256, 6), // v_cmp_tru_f32 v6, v6 - EncodeVopc(0x07, 6 + 256, 6), // v_cmp_o_f32 v6, v6 - EncodeVopc(0x08, 6 + 256, 6), // v_cmp_u_f32 v6, v6 - EncodeVopc(0x09, 6 + 256, 6), // v_cmp_nge_f32 v6, v6 - EncodeVopc(0x0a, 6 + 256, 6), // v_cmp_nlg_f32 v6, v6 - EncodeVopc(0x0b, 6 + 256, 6), // v_cmp_ngt_f32 v6, v6 - EncodeVopc(0x0c, 6 + 256, 6), // v_cmp_nle_f32 v6, v6 - EncodeVopc(0x0d, 6 + 256, 6), // v_cmp_neq_f32 v6, v6 - 0x7c1a02f9u, - 0x068680f0u, // v_cmp_neq_f32 s0, 0.5, v1 (SDWA) - EncodeVopc(0xd1, 249, 6), - EncodeVopcSdwa(5, 0, 0, 4), // v_cmpx_lt_u32 exec, v5, v6 (SDWA) - EncodeVopc(0x0e, 6 + 256, 6), // v_cmp_nlt_f32 v6, v6 - EncodeVopc(0x19, 6 + 256, 6), // v_cmpx_nge_f32 v6, v6 - EncodeVopc(0x1a, 6 + 256, 6), // v_cmpx_nlg_f32 v6, v6 - EncodeVopc(0x1b, 6 + 256, 6), // v_cmpx_ngt_f32 v6, v6 - EncodeVopc(0x1c, 6 + 256, 6), // v_cmpx_nle_f32 v6, v6 - EncodeVopc(0x1d, 6 + 256, 6), // v_cmpx_neq_f32 v6, v6 - EncodeVopc(0x1e, 6 + 256, 6), // v_cmpx_nlt_f32 v6, v6 - EncodeVopc(0x80, 5 + 256, 5), // v_cmp_f_i32 v5, v5 - EncodeVopc(0x87, 5 + 256, 5), // v_cmp_t_i32 v5, v5 - EncodeVopc(0xc0, 5 + 256, 5), // v_cmp_f_u32 v5, v5 - EncodeVopc(0xc7, 5 + 256, 5), // v_cmp_t_u32 v5, v5 - 0xd4e5006au, - 0x0000d47eu, // v_cmp_ne_u64 vcc, exec, vcc - EncodeVop3Word0(0x141, 8), - EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), - 0xd5410004u, - 0x20121301u, // v_mad_f32 v4, -v1, v9, s4 - EncodeVop3Word0(0x144, 110), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x145, 111), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x146, 112), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x147, 113), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - 0xd5470005u, - 0x841a0102u, // v_cubema_f32 v5, v2, v0, -v6 - EncodeVop3Word0(0x14b, 37), - EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), - EncodeVop3Word0(0x151, 38), - EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), - EncodeVop3Word0(0x154, 39), - EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), - EncodeVop3Word0(0x157, 40), - EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), - EncodeVop3Word0(0x152, 41), - EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), - EncodeVop3Word0(0x153, 42), - EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), - EncodeVop3Word0(0x155, 43), - EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), - EncodeVop3Word0(0x156, 44), - EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), - EncodeVop3Word0(0x158, 45), - EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), - EncodeVop3Word0(0x159, 46), - EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), - EncodeVop3Word0(0x148, 47), - EncodeVop3Word1(5 + 256, 129, 132), - EncodeVop3Word0(0x149, 48), - EncodeVop3Word1(5 + 256, 129, 132), - EncodeVop3Word0(0x14a, 49), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x14e, 50), - EncodeVop3Word1(5 + 256, 6 + 256, 129), - EncodeVop3Word0(0x36d, 51), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x169, 52), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x16a, 53), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x16c, 114), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x371, 54), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x372, 55), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x178, 56), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x346, 57), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x347, 58), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x345, 59), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x36f, 60), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x15d, 61), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x142, 62), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x143, 63), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), - EncodeVop3Word0(0x16b, 69), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x30f, 70), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x310, 71), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x319, 72), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - 0xd70a1007u, - 0x00020affu, - 0x0000ffffu, // v_max_i16 v7, 0xffff, v5.hi - 0xd70c5007u, - 0x00020882u, // v_min_i16 v7, 2, v4.hi - EncodeVop3Word0(0x363, 73), - EncodeVop3Word1(129, 132, 0), - EncodeVop3Word0(0x360, 26), - EncodeVop3Word1(5 + 256, 130, 0), - EncodeVop3Word0(0x361, 107), - EncodeVop3Word1(5 + 256, 130, 0), - EncodeVop3Word0(0x377, 108), - EncodeVop3Word1(5 + 256, 128, 128), - EncodeVop3Word0(0x378, 109), - EncodeVop3Word1(5 + 256, 128, 128), - EncodeVop3Word0(0x12f, 75), - EncodeVop3Word1(6 + 256, 6 + 256, 0), - 0xd52f0000u, - 0x60020300u, // v_cvt_pkrtz_f16_f32 v0, -v0, -v1 - EncodeVop3Word0(0x362, 76), - EncodeVop3Word1(6 + 256, 129, 0), - 0xd7620107u, - 0x00018509u, // v_ldexp_f32 v7, abs(v9), -2 - 0xd762800du, - 0x00018906u, // v_ldexp_f32 clamp v13, v6, -4 - EncodeVop3Word0(0x364, 77), - EncodeVop3Word1(5 + 256, 6 + 256, 0), - EncodeVop3Word0(0x368, 78), - EncodeVop3Word1(6 + 256, 6 + 256, 0), - EncodeVop3Word0(0x369, 79), - EncodeVop3Word1(6 + 256, 6 + 256, 0), - EncodeVop3Word0(0x36a, 80), - EncodeVop3Word1(5 + 256, 7 + 256, 0), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopk(0x00, 9, 7), // s_movk_i32 s9, 7 + EncodeSopk(0x0b, 9, 3), // s_cmp_gt_u32 s9, 3 + EncodeVop1(0x00, 0, 0), // v_nop + EncodeVop1(0x01, 5, 9), // v_mov_b32 v5, s9 + EncodeVop1(0x01, 103, 249), + EncodeVop1Sdwa(5, 6, 0, 4), // v_mov_b32 v103, v5 word0 + EncodeVop1(0x06, 104, 249), + EncodeVop1Sdwa(103, 6, 0, 4), // v_cvt_f32_u32 v104, v103 word0 + EncodeVop1(0x0a, 105, 249), + EncodeVop1Sdwa(6, 5, 2, 6), // v_cvt_f16_f32 v105.hi, v6 + EncodeVop1(0x01, 106, 250), + EncodeVop1Dpp(5), // v_mov_b32 v106, v5 dpp + EncodeVop1(0x02, 24, 5 + 256), // v_readfirstlane_b32 s24, v5 + EncodeVop1(0x06, 6, 5 + 256), // v_cvt_f32_u32 v6, v5 + EncodeVop1(0x07, 7, 6 + 256), // v_cvt_u32_f32 v7, v6 + EncodeVop1(0x05, 9, 5 + 256), // v_cvt_f32_i32 v9, v5 + EncodeVop1(0x08, 10, 6 + 256), // v_cvt_i32_f32 v10, v6 + EncodeVop1(0x0a, 99, 6 + 256), // v_cvt_f16_f32 v99, v6 + EncodeVop1(0x0b, 100, 99 + 256), // v_cvt_f32_f16 v100, v99 + 0x7e1016f9u, + 0x00250602u, // v_cvt_f32_f16 v8, abs(v2.hi) + EncodeVop1(0x0d, 64, 6 + 256), // v_cvt_flr_i32_f32 v64, v6 + EncodeVop1(0x0e, 81, 5 + 256), // v_cvt_off_f32_i4 v81, v5 + EncodeVop1(0x11, 65, 5 + 256), // v_cvt_f32_ubyte0 v65, v5 + EncodeVop1(0x12, 66, 5 + 256), // v_cvt_f32_ubyte1 v66, v5 + EncodeVop1(0x13, 67, 5 + 256), // v_cvt_f32_ubyte2 v67, v5 + EncodeVop1(0x14, 68, 5 + 256), // v_cvt_f32_ubyte3 v68, v5 + EncodeVop1(0x2a, 11, 6 + 256), // v_rcp_f32 v11, v6 + EncodeVop1(0x20, 12, 6 + 256), // v_fract_f32 v12, v6 + EncodeVop1(0x21, 13, 6 + 256), // v_trunc_f32 v13, v6 + EncodeVop1(0x22, 14, 6 + 256), // v_ceil_f32 v14, v6 + EncodeVop1(0x23, 15, 6 + 256), // v_rndne_f32 v15, v6 + EncodeVop1(0x24, 16, 6 + 256), // v_floor_f32 v16, v6 + EncodeVop1(0x25, 17, 6 + 256), // v_exp_f32 v17, v6 + EncodeVop1(0x27, 18, 6 + 256), // v_log_f32 v18, v6 + EncodeVop1(0x2e, 19, 6 + 256), // v_rsq_f32 v19, v6 + EncodeVop1(0x33, 20, 6 + 256), // v_sqrt_f32 v20, v6 + EncodeVop1(0x35, 21, 6 + 256), // v_sin_f32 v21, v6 + EncodeVop1(0x36, 22, 6 + 256), // v_cos_f32 v22, v6 + EncodeVop1(0x37, 27, 5 + 256), // v_not_b32 v27, v5 + EncodeVop1(0x38, 28, 5 + 256), // v_bfrev_b32 v28, v5 + EncodeVop1(0x39, 31, 5 + 256), // v_ffbh_u32 v31, v5 + EncodeVop1(0x3a, 32, 5 + 256), // v_ffbl_b32 v32, v5 + 0x7e6e870cu, // v_movrels_b32 v55, v12 + EncodeVop2(0x1e, 83, 5 + 256, 6), // v_xnor_b32 v83, v5, v6 + EncodeVop2(0x23, 84, 5 + 256, 6), // v_mbcnt_lo_u32_b32 v84, v5, v6 + EncodeVop2(0x24, 85, 5 + 256, 6), // v_mbcnt_hi_u32_b32 v85, v5, v6 + EncodeVop2(0x1f, 89, 6 + 256, 6), // v_mac_f32 v89, v6, v6 + EncodeVop2(0x20, 90, 6 + 256, 5), + 0x3f800000u, // v_madmk_f32 v90, v6, 1.0, v5 + EncodeVop2(0x21, 91, 6 + 256, 5), + 0x40000000u, // v_madak_f32 v91, v6, v5, 2.0 + EncodeVop2(0x2b, 92, 6 + 256, 6), // v_mac_f32 v92, v6, v6 + EncodeVop2(0x2c, 93, 6 + 256, 5), + 0x3f000000u, // v_madmk_f32 v93, v6, 0.5, v5 + EncodeVop2(0x2d, 94, 6 + 256, 5), + 0x40400000u, // v_madak_f32 v94, v6, v5, 3.0 + EncodeVop2(0x2f, 95, 6 + 256, 6), // v_cvt_pkrtz_f16_f32 v95, v6, v6 + EncodeVop2(0x02, 102, 95 + 256, 95), // v_dot2c_f32_f16 v102, v95, v95 + EncodeVop2(0x28, 97, 5 + 256, 6), // v_addc_u32 v97, vcc, v5, v6, vcc + EncodeVop2(0x25, 123, 249, 6), + EncodeVop2Sdwa(5, 6, 0, 4, 6), + EncodeVop2(0x1b, 124, 249, 6), + EncodeVop2Sdwa(5, 6, 0, 4, 5), + 0x025e6af9u, + 0x16060635u, // v_cndmask_b32 v47, v53, -v53 + 0x020e02f9u, + 0x06040600u, // v_cndmask_b32 v7, v0.lo, v1; full-width destination + 0x100490f9u, + 0x86860600u, // v_mul_f32 v2, s0, s72 (SDWA full) + EncodeVop2(0x35, 9, 249, 1), + EncodeVop2Sdwa(0, 4, 2, 4, 4), + 0x660000f9u, + EncodeVop2Sdwa(0, 4, 2, 4, 4), + EncodeVop2(0x03, 125, 250, 6), + EncodeVop2Dpp(5), + EncodeVop3Word0(0x103, 35), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0(0x10b, 36), + EncodeVop3Word1(5 + 256, 5 + 256, 0), + EncodeVop3Word0(0x11e, 86), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x11f, 96), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0Sdst(0x128, 98, 30), + EncodeVop3Word1(5 + 256, 6 + 256, 106), + EncodeVop3Word0(0x123, 87), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x124, 88), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x180, 0), + EncodeVop3Word1(0, 0, 0), + EncodeVop3Word0(0x181, 23), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x182, 25), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x185, 24), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x18a, 101), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x18d, 74), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x18e, 82), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1a0, 25), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x1aa, 26), + EncodeVop3Word1(6 + 256, 0, 0), + EncodeVop3Word0(0x1b7, 29), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1b8, 30), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1b9, 33), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVop3Word0(0x1ba, 34), + EncodeVop3Word1(5 + 256, 0, 0), + EncodeVopc(0x04, 6 + 256, 6), // v_cmp_gt_f32 v6, v6 + EncodeVopc(0xc4, 7 + 256, 5), // v_cmp_gt_u32 v7, v5 + EncodeVopc(0x14, 6 + 256, 6), // v_cmpx_gt_f32 v6, v6 + EncodeVopc(0xd4, 7 + 256, 5), // v_cmpx_gt_u32 v7, v5 + EncodeVopc(0x00, 6 + 256, 6), // v_cmp_f_f32 v6, v6 + EncodeVopc(0x0f, 6 + 256, 6), // v_cmp_tru_f32 v6, v6 + EncodeVopc(0x07, 6 + 256, 6), // v_cmp_o_f32 v6, v6 + EncodeVopc(0x08, 6 + 256, 6), // v_cmp_u_f32 v6, v6 + EncodeVopc(0x09, 6 + 256, 6), // v_cmp_nge_f32 v6, v6 + EncodeVopc(0x0a, 6 + 256, 6), // v_cmp_nlg_f32 v6, v6 + EncodeVopc(0x0b, 6 + 256, 6), // v_cmp_ngt_f32 v6, v6 + EncodeVopc(0x0c, 6 + 256, 6), // v_cmp_nle_f32 v6, v6 + EncodeVopc(0x0d, 6 + 256, 6), // v_cmp_neq_f32 v6, v6 + 0x7c1a02f9u, + 0x068680f0u, // v_cmp_neq_f32 s0, 0.5, v1 (SDWA) + EncodeVopc(0xd1, 249, 6), + EncodeVopcSdwa(5, 0, 0, 4), // v_cmpx_lt_u32 exec, v5, v6 (SDWA) + EncodeVopc(0x0e, 6 + 256, 6), // v_cmp_nlt_f32 v6, v6 + EncodeVopc(0x19, 6 + 256, 6), // v_cmpx_nge_f32 v6, v6 + EncodeVopc(0x1a, 6 + 256, 6), // v_cmpx_nlg_f32 v6, v6 + EncodeVopc(0x1b, 6 + 256, 6), // v_cmpx_ngt_f32 v6, v6 + EncodeVopc(0x1c, 6 + 256, 6), // v_cmpx_nle_f32 v6, v6 + EncodeVopc(0x1d, 6 + 256, 6), // v_cmpx_neq_f32 v6, v6 + EncodeVopc(0x1e, 6 + 256, 6), // v_cmpx_nlt_f32 v6, v6 + EncodeVopc(0x80, 5 + 256, 5), // v_cmp_f_i32 v5, v5 + EncodeVopc(0x87, 5 + 256, 5), // v_cmp_t_i32 v5, v5 + EncodeVopc(0xc0, 5 + 256, 5), // v_cmp_f_u32 v5, v5 + EncodeVopc(0xc7, 5 + 256, 5), // v_cmp_t_u32 v5, v5 + 0xd4e5006au, + 0x0000d47eu, // v_cmp_ne_u64 vcc, exec, vcc + EncodeVop3Word0(0x141, 8), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + 0xd5410004u, + 0x20121301u, // v_mad_f32 v4, -v1, v9, s4 + EncodeVop3Word0(0x144, 110), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x145, 111), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x146, 112), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x147, 113), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + 0xd5470005u, + 0x841a0102u, // v_cubema_f32 v5, v2, v0, -v6 + EncodeVop3Word0(0x14b, 37), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x151, 38), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x154, 39), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x157, 40), + EncodeVop3Word1(6 + 256, 6 + 256, 6 + 256), + EncodeVop3Word0(0x152, 41), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x153, 42), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x155, 43), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x156, 44), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x158, 45), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x159, 46), + EncodeVop3Word1(5 + 256, 5 + 256, 5 + 256), + EncodeVop3Word0(0x148, 47), + EncodeVop3Word1(5 + 256, 129, 132), + EncodeVop3Word0(0x149, 48), + EncodeVop3Word1(5 + 256, 129, 132), + EncodeVop3Word0(0x14a, 49), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x14e, 50), + EncodeVop3Word1(5 + 256, 6 + 256, 129), + EncodeVop3Word0(0x36d, 51), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x169, 52), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x16a, 53), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x16c, 114), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x371, 54), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x372, 55), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x178, 56), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x346, 57), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x347, 58), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x345, 59), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x36f, 60), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x15d, 61), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x142, 62), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x143, 63), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), + EncodeVop3Word0(0x16b, 69), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x30f, 70), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x310, 71), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x319, 72), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + 0xd70a1007u, + 0x00020affu, + 0x0000ffffu, // v_max_i16 v7, 0xffff, v5.hi + 0xd70c5007u, + 0x00020882u, // v_min_i16 v7, 2, v4.hi + EncodeVop3Word0(0x363, 73), + EncodeVop3Word1(129, 132, 0), + EncodeVop3Word0(0x360, 26), + EncodeVop3Word1(5 + 256, 130, 0), + EncodeVop3Word0(0x361, 107), + EncodeVop3Word1(5 + 256, 130, 0), + EncodeVop3Word0(0x377, 108), + EncodeVop3Word1(5 + 256, 128, 128), + EncodeVop3Word0(0x378, 109), + EncodeVop3Word1(5 + 256, 128, 128), + EncodeVop3Word0(0x12f, 75), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + 0xd52f0000u, + 0x60020300u, // v_cvt_pkrtz_f16_f32 v0, -v0, -v1 + EncodeVop3Word0(0x362, 76), + EncodeVop3Word1(6 + 256, 129, 0), + 0xd7620107u, + 0x00018509u, // v_ldexp_f32 v7, abs(v9), -2 + 0xd762800du, + 0x00018906u, // v_ldexp_f32 clamp v13, v6, -4 + EncodeVop3Word0(0x364, 77), + EncodeVop3Word1(5 + 256, 6 + 256, 0), + EncodeVop3Word0(0x368, 78), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0(0x369, 79), + EncodeVop3Word1(6 + 256, 6 + 256, 0), + EncodeVop3Word0(0x36a, 80), + EncodeVop3Word1(5 + 256, 7 + 256, 0), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_movk_i32 s9"), - "new decoder did not decode SOPK mov"); - Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32"), - "new decoder did not decode SOPK compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_nop"), - "new decoder did not decode old-backed VOP1 no-op"); - Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v5"), - "new decoder did not decode VOP1 mov"); - Check(Common::ContainsStr(result.decoded_dump, "v_movrels_b32 v55, v12"), - "new decoder did not decode VOP1 V_MOVRELS_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v103, v5.sdwa(sel=4"), - "new decoder did not decode VOP1 SDWA source selector"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v105.sdwa(sel=5"), - "new decoder did not decode VOP1 SDWA destination selector"); - Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v106, v5.dpp"), - "new decoder did not decode VOP1 DPP source metadata"); - Check(!Common::ContainsStr(result.decoded_dump, "VOP1 SDWA/DPP modifiers are not implemented"), - "new decoder still reports blanket VOP1 SDWA/DPP unsupported reason"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v123, v5.sdwa(sel=4"), - "new decoder did not decode VOP2 SDWA source selector"); - Check(Common::ContainsStr(result.decoded_dump, "v_and_b32 v124, v5.sdwa(sel=4"), - "new decoder did not decode VOP2 SDWA first source selector"); - Check(Common::ContainsStr(result.decoded_dump, "v6.sdwa(sel=5"), - "new decoder did not decode VOP2 SDWA second source selector"); - Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v47, v53,") && - Common::ContainsStr(result.decoded_dump, "v53.neg"), - "new decoder did not decode V_CNDMASK_B32 SDWA source modifier"); - Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v7, v0.sdwa(sel=4") && - Common::ContainsStr(result.decoded_dump, "v1"), - "new decoder did not decode full-destination V_CNDMASK_B32 with SDWA source"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v125, v5.dpp"), - "new decoder did not decode VOP2 DPP source metadata"); - Check(!Common::ContainsStr(result.decoded_dump, "VOP2 SDWA/DPP modifiers are not implemented"), - "new decoder still reports blanket VOP2 SDWA/DPP unsupported reason"); - Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s24, v5"), - "new decoder did not decode old-backed V_READFIRSTLANE_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_u32 v6"), - "new decoder did not decode VOP1 conversion"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_i32 v9"), - "new decoder did not decode VOP1 signed int-to-float conversion"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_i32_f32 v10"), - "new decoder did not decode VOP1 float-to-signed-int conversion"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v99"), - "new decoder did not decode old-backed V_CVT_F16_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_f16 v100"), - "new decoder did not decode old-backed native V_CVT_F32_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_f16 v8, v2.sdwa(sel=5") && - Common::ContainsStr(result.decoded_dump, "v2.sdwa(sel=5,sext=0).abs"), - "new decoder did not decode V_CVT_F32_F16 SDWA source selector modifier"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_flr_i32_f32 v64"), - "new decoder did not decode old-backed V_CVT_FLR_I32_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_off_f32_i4 v81"), - "new decoder did not decode old-backed V_CVT_OFF_F32_I4"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte0 v65"), - "new decoder did not decode old-backed V_CVT_F32_UBYTE0"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte1 v66"), - "new decoder did not decode old-backed V_CVT_F32_UBYTE1"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte2 v67"), - "new decoder did not decode old-backed V_CVT_F32_UBYTE2"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte3 v68"), - "new decoder did not decode old-backed V_CVT_F32_UBYTE3"); - Check(Common::ContainsStr(result.decoded_dump, "v_rcp_f32 v11"), - "new decoder did not decode VOP1 reciprocal"); - Check(Common::ContainsStr(result.decoded_dump, "v_fract_f32 v12"), - "new decoder did not decode VOP1 fract"); - Check(Common::ContainsStr(result.decoded_dump, "v_cos_f32 v22"), - "new decoder did not decode VOP1 cosine"); - Check(Common::ContainsStr(result.decoded_dump, "v_not_b32 v27"), - "new decoder did not decode VOP1 not"); - Check(Common::ContainsStr(result.decoded_dump, "v_bfrev_b32 v28"), - "new decoder did not decode VOP1 bit reverse"); - Check(Common::ContainsStr(result.decoded_dump, "v_ffbh_u32 v31"), - "new decoder did not decode VOP1 find-first-bit-high"); - Check(Common::ContainsStr(result.decoded_dump, "v_ffbl_b32 v32"), - "new decoder did not decode VOP1 find-first-bit-low"); - Check(Common::ContainsStr(result.decoded_dump, "v_xnor_b32 v83, v5, v6"), - "new decoder did not decode old-backed V_XNOR_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_lo_u32_b32 v84, v5, v6"), - "new decoder did not decode old-backed V_MBCNT_LO_U32_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_hi_u32_b32 v85, v5, v6"), - "new decoder did not decode old-backed V_MBCNT_HI_U32_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v89, v6, v6"), - "new decoder did not decode old-backed V_MAC_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_madmk_f32 v90, v6, 0x3f800000, v5"), - "new decoder did not decode old-backed V_MADMK_F32 literal form"); - Check(Common::ContainsStr(result.decoded_dump, "v_madak_f32 v91, v6, v5, 0x40000000"), - "new decoder did not decode old-backed V_MADAK_F32 literal form"); - Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v92, v6, v6"), - "new decoder did not decode old-backed alternate V_MAC_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_madmk_f32 v93, v6, 0x3f000000, v5"), - "new decoder did not decode old-backed alternate V_MADMK_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_madak_f32 v94, v6, v5, 0x40400000"), - "new decoder did not decode old-backed alternate V_MADAK_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pkrtz_f16_f32 v95, v6, v6"), - "new decoder did not decode old-backed native V_CVT_PKRTZ_F16_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_dot2c_f32_f16 v102, v95, v95"), - "new decoder did not decode old-backed V_DOT2C_F32_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_addc_u32 v97, vcc_lo, v5, v6, vcc_lo"), - "new decoder did not decode old-backed V_ADD_CO_U32 carry form"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v35"), - "new decoder did not decode VOP3-encoded VOP2 float add"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_u32_u24 v36"), - "new decoder did not decode VOP3-encoded VOP2 24-bit multiply"); - Check(Common::ContainsStr(result.decoded_dump, "v_xnor_b32 v86, v5, v6"), - "new decoder did not decode old-backed VOP3-encoded V_XNOR_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v96, v6, v6"), - "new decoder did not decode old-backed VOP3-encoded V_MAC_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_addc_u32 v98, s30, v5, v6, vcc_lo"), - "new decoder did not decode old-backed VOP3 V_ADD_CO_U32 carry form"); - Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_lo_u32_b32 v87, v5, v6"), - "new decoder did not decode old-backed VOP3-encoded V_MBCNT_LO_U32_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mbcnt_hi_u32_b32 v88, v5, v6"), - "new decoder did not decode old-backed VOP3-encoded V_MBCNT_HI_U32_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v23"), - "new decoder did not decode VOP3-encoded VOP1 move"); - Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), - "new decoder did not decode old-backed VOP3-encoded V_READFIRSTLANE_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_i32 v24"), - "new decoder did not decode VOP3-encoded VOP1 signed conversion"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v101"), - "new decoder did not decode old-backed VOP3-encoded V_CVT_F16_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_flr_i32_f32 v74"), - "new decoder did not decode old-backed VOP3-encoded V_CVT_FLR_I32_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_off_f32_i4 v82"), - "new decoder did not decode old-backed VOP3-encoded V_CVT_OFF_F32_I4"); - Check(Common::ContainsStr(result.decoded_dump, "v_fract_f32 v25"), - "new decoder did not decode VOP3-encoded VOP1 fract"); - Check(Common::ContainsStr(result.decoded_dump, "v_rcp_f32 v26"), - "new decoder did not decode VOP3-encoded VOP1 reciprocal"); - Check(Common::ContainsStr(result.decoded_dump, "v_not_b32 v29"), - "new decoder did not decode VOP3-encoded VOP1 not"); - Check(Common::ContainsStr(result.decoded_dump, "v_bfrev_b32 v30"), - "new decoder did not decode VOP3-encoded VOP1 bit reverse"); - Check(Common::ContainsStr(result.decoded_dump, "v_ffbh_u32 v33"), - "new decoder did not decode VOP3-encoded VOP1 find-first-bit-high"); - Check(Common::ContainsStr(result.decoded_dump, "v_ffbl_b32 v34"), - "new decoder did not decode VOP3-encoded VOP1 find-first-bit-low"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_gt_f32"), - "new decoder did not decode VOPC float compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_f32"), - "new decoder did not decode VOPC float compare-and-mask"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_u32"), - "new decoder did not decode VOPC uint compare-and-mask"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_f32"), - "new decoder did not decode old-backed V_CMP_F_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_tru_f32"), - "new decoder did not decode old-backed V_CMP_TRU_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_o_f32"), - "new decoder did not decode old-backed V_CMP_O_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_u_f32"), - "new decoder did not decode old-backed V_CMP_U_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nge_f32"), - "new decoder did not decode old-backed V_CMP_NGE_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nlg_f32"), - "new decoder did not decode old-backed V_CMP_NLG_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ngt_f32"), - "new decoder did not decode old-backed V_CMP_NGT_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nle_f32"), - "new decoder did not decode old-backed V_CMP_NLE_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_neq_f32"), - "new decoder did not decode old-backed V_CMP_NEQ_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_neq_f32 s0, 0.500000, v1"), - "new decoder did not decode old-backed V_CMP_NEQ_F32 SDWA scalar destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_lt_u32 exec_lo, v5.sdwa(sel=4") && - Common::ContainsStr(result.ir_dump, "CompareMaskLtU32 exec_lo"), - "new decoder did not route V_CMPX SDWA destination to exec"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nlt_f32"), - "new decoder did not decode old-backed V_CMP_NLT_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nge_f32"), - "new decoder did not decode old-backed V_CMPX_NGE_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nlg_f32"), - "new decoder did not decode old-backed V_CMPX_NLG_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_ngt_f32"), - "new decoder did not decode old-backed V_CMPX_NGT_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nle_f32"), - "new decoder did not decode old-backed V_CMPX_NLE_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_neq_f32"), - "new decoder did not decode old-backed V_CMPX_NEQ_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nlt_f32"), - "new decoder did not decode old-backed V_CMPX_NLT_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_i32"), - "new decoder did not decode old-backed V_CMP_F_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_t_i32"), - "new decoder did not decode old-backed V_CMP_T_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_u32"), - "new decoder did not decode old-backed V_CMP_F_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_t_u32"), - "new decoder did not decode old-backed V_CMP_T_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ne_u64 vcc_lo, exec_lo, vcc_lo"), - "new decoder did not decode VOP3-encoded V_CMP_NE_U64"); - Check(Common::ContainsStr(result.ir_dump, "CompareNeU64 vcc_lo, exec_lo, vcc_lo"), - "V_CMP_NE_U64 did not lower to 64-bit compare IR"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_f32 v8"), - "new decoder did not decode VOP3 mad"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_f32 v4, v1.neg, v9, s4"), - "new decoder did not decode VOP3 mad source modifiers"); - Check(Common::ContainsStr(result.decoded_dump, "v_cubeid_f32 v110, v5, v6, v7"), - "new decoder did not decode old-backed V_CUBEID_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cubesc_f32 v111, v5, v6, v7"), - "new decoder did not decode old-backed V_CUBESC_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cubetc_f32 v112, v5, v6, v7"), - "new decoder did not decode old-backed V_CUBETC_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cubema_f32 v113, v5, v6, v7"), - "new decoder did not decode old-backed V_CUBEMA_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cubema_f32 v5, v2, v0, v6.neg"), - "new decoder did not decode V_CUBEMA_F32 source modifier"); - Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v37"), - "new decoder did not decode VOP3 fma"); - Check(Common::ContainsStr(result.decoded_dump, "v_min3_f32 v38"), - "new decoder did not decode VOP3 min3"); - Check(Common::ContainsStr(result.decoded_dump, "v_max3_f32 v39"), - "new decoder did not decode VOP3 max3"); - Check(Common::ContainsStr(result.decoded_dump, "v_med3_f32 v40"), - "new decoder did not decode VOP3 med3"); - Check(Common::ContainsStr(result.decoded_dump, "v_min3_i32 v41"), - "new decoder did not decode VOP3 signed min3"); - Check(Common::ContainsStr(result.decoded_dump, "v_min3_u32 v42"), - "new decoder did not decode VOP3 unsigned min3"); - Check(Common::ContainsStr(result.decoded_dump, "v_max3_i32 v43"), - "new decoder did not decode VOP3 signed max3"); - Check(Common::ContainsStr(result.decoded_dump, "v_max3_u32 v44"), - "new decoder did not decode VOP3 unsigned max3"); - Check(Common::ContainsStr(result.decoded_dump, "v_med3_i32 v45"), - "new decoder did not decode VOP3 signed med3"); - Check(Common::ContainsStr(result.decoded_dump, "v_med3_u32 v46"), - "new decoder did not decode VOP3 unsigned med3"); - Check(Common::ContainsStr(result.decoded_dump, "v_bfe_u32 v47"), - "new decoder did not decode VOP3 unsigned bitfield extract"); - Check(Common::ContainsStr(result.decoded_dump, "v_bfe_i32 v48"), - "new decoder did not decode VOP3 signed bitfield extract"); - Check(Common::ContainsStr(result.decoded_dump, "v_bfi_b32 v49"), - "new decoder did not decode VOP3 bitfield insert-select"); - Check(Common::ContainsStr(result.decoded_dump, "v_alignbit_b32 v50"), - "new decoder did not decode VOP3 alignbit"); - Check(Common::ContainsStr(result.decoded_dump, "v_add3_u32 v51"), - "new decoder did not decode VOP3 add3"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_lo_u32 v52, v5, v6"), - "new decoder did not decode old-backed V_MUL_LO_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_hi_u32 v53, v5, v6"), - "new decoder did not decode old-backed V_MUL_HI_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_hi_i32 v114, v5, v6"), - "new decoder did not decode RDNA2 V_MUL_HI_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_and_or_b32 v54, v5, v6, v7"), - "new decoder did not decode old-backed V_AND_OR_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_or3_b32 v55, v5, v6, v7"), - "new decoder did not decode old-backed V_OR3_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_xor3_b32 v56, v5, v6, v7"), - "new decoder did not decode old-backed V_XOR3_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_lshl_add_u32 v57, v5, v6, v7"), - "new decoder did not decode old-backed V_LSHL_ADD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_lshl_u32 v58, v5, v6, v7"), - "new decoder did not decode old-backed V_ADD_LSHL_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_xad_u32 v59, v5, v6, v7"), - "new decoder did not decode old-backed V_XAD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_lshl_or_b32 v60, v5, v6, v7"), - "new decoder did not decode old-backed V_LSHL_OR_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_sad_u32 v61, v5, v6, v7"), - "new decoder did not decode old-backed V_SAD_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_i32_i24 v62, v5, v6, v7"), - "new decoder did not decode old-backed V_MAD_I32_I24"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_u32_u24 v63, v5, v6, v7"), - "new decoder did not decode old-backed V_MAD_U32_U24"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_lo_i32 v69, v5, v6"), - "new decoder did not decode old-backed V_MUL_LO_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_i32 v70, s0, v5, v6"), - "new decoder did not decode old-backed V_ADD_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_sub_i32 v71, s0, v5, v6"), - "new decoder did not decode RDNA2 V_SUB_CO_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_subrev_i32 v72, s0, v5, v6"), - "new decoder did not decode old-backed V_SUBREV_I32"); - Check(Common::ContainsStr( - result.decoded_dump, - "v_max_i16 v7.sdwa(sel=4,sext=0), 0x0000ffff, v5.opsel(lo=1,hi=0,neghi=0)"), - "new decoder did not decode RDNA2 V_MAX_I16 literal/op_sel form"); - Check(Common::ContainsStr(result.decoded_dump, - "v_min_i16 v7.sdwa(sel=5,sext=0), 2, v4.opsel(lo=1,hi=0,neghi=0)"), - "new decoder did not decode RDNA2 V_MIN_I16 op_sel form"); - Check(Common::ContainsStr(result.decoded_dump, "v_bfm_b32 v73, 1, 4"), - "new decoder did not decode old-backed V_BFM_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), - "new decoder did not decode old-backed V_READLANE_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_writelane_b32 v107, v5, 2"), - "new decoder did not decode old-backed V_WRITELANE_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_permlane16_b32 v108, v5, 0, 0"), - "new decoder did not decode old-backed V_PERMLANE16_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_permlanex16_b32 v109, v5, 0, 0"), - "new decoder did not decode old-backed V_PERMLANEX16_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pkrtz_f16_f32 v75, v6, v6"), - "new decoder did not decode old-backed V_CVT_PKRTZ_F16_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pkrtz_f16_f32 v0, v0.neg, v1.neg"), - "new decoder did not decode V_CVT_PKRTZ_F16_F32 source modifiers"); - Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v76, v6, 1"), - "new decoder did not decode old-backed V_LDEXP_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v7, v9.abs, -2"), - "new decoder did not decode V_LDEXP_F32 source modifier"); - Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v13.clamp, v6, -4"), - "new decoder did not decode V_LDEXP_F32 clamp modifier"); - Check(Common::ContainsStr(result.decoded_dump, "v_bcnt_u32_b32 v77, v5, v6"), - "new decoder did not decode old-backed VOP3 V_BCNT_U32_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pknorm_i16_f32 v78, v6, v6"), - "new decoder did not decode old-backed V_CVT_PKNORM_I16_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pknorm_u16_f32 v79, v6, v6"), - "new decoder did not decode old-backed V_CVT_PKNORM_U16_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_pk_u16_u32 v80, v5, v7"), - "new decoder did not decode old-backed V_CVT_PK_U16_U32"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_f32 v2, s0, s72"), - "new decoder did not decode old-backed V_MUL_F32 SDWA full-width form"); - Check(Common::ContainsStr(result.decoded_dump, "v_mul_f16 v9.sdwa(sel=4") && - Common::ContainsStr(result.decoded_dump, "v0.sdwa(sel=4") && - Common::ContainsStr(result.decoded_dump, "v1.sdwa(sel=4"), - "new decoder did not decode V_MUL_F16 SDWA low-half form"); - Check(Common::ContainsStr(result.decoded_dump, "v_sub_f16 v0.sdwa(sel=4") && - Common::ContainsStr(result.decoded_dump, "v0.sdwa(sel=4"), - "new decoder did not decode V_SUB_F16 SDWA low-half form"); - Check(Common::ContainsStr(result.ir_dump, "ConvertU32ToF32 v6"), - "VOP1 uint-to-float conversion did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 v103, v5.sdwa(sel=4"), - "VOP1 SDWA source selector did not lower to IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToF16 v105.sdwa(sel=5"), - "VOP1 SDWA destination selector did not lower to IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 v106.dpp") && - Common::ContainsStr(result.ir_dump, "v5.dpp"), - "VOP1 DPP source/destination did not lower to IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "MoveRelSourceU32 v55, v12, m0"), - "V_MOVRELS_B32 did not lower to indexed VGPR-source IR"); - Check(Common::ContainsStr(result.ir_dump, "IAddU32 v123, v5.sdwa(sel=4"), - "VOP2 SDWA did not lower first source metadata to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU32 v124, v5.sdwa(sel=4"), - "VOP2 SDWA bitwise op did not lower first source metadata to IR"); - Check(Common::ContainsStr(result.ir_dump, "v6.sdwa(sel=5"), - "VOP2 SDWA bitwise op did not lower second source metadata to IR"); - Check(Common::ContainsStr(result.ir_dump, "SelectMaskF32Bits v47, vcc_lo, v53.neg, v53") && - Common::ContainsStr(result.ir_dump, "v53.neg"), - "V_CNDMASK_B32 SDWA source modifier did not lower to float-bit select IR"); - Check(Common::ContainsStr(result.ir_dump, "SelectMaskU32 v7, vcc_lo, v1, v0.sdwa(sel=4"), - "full-destination V_CNDMASK_B32 with SDWA source did not lower to integer select IR"); - Check(Common::ContainsStr(result.ir_dump, "FAddF32 v125.dpp") && - Common::ContainsStr(result.ir_dump, "v5.dpp"), - "VOP2 DPP source/destination did not lower to IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "FMulF32 v2, s0, s72"), - "V_MUL_F32 SDWA full-width form did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "MulF16 v9.sdwa(sel=4") && - Common::ContainsStr(result.ir_dump, "v0.sdwa(sel=4") && - Common::ContainsStr(result.ir_dump, "v1.sdwa(sel=4"), - "V_MUL_F16 SDWA low-half form did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "SubF16 v0.sdwa(sel=4") && - Common::ContainsStr(result.ir_dump, "v0.sdwa(sel=4"), - "V_SUB_F16 SDWA low-half form did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToU32 v7"), - "VOP1 float-to-uint conversion did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertI32ToF32 v9"), - "VOP1 signed int-to-float conversion did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToI32 v10"), - "VOP1 float-to-signed-int conversion did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToF16 v99, v6"), - "V_CVT_F16_F32 did not lower to shared half-pack IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToF32 v100, v99"), - "V_CVT_F32_F16 did not lower to shared half-unpack IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToF32 v8, v2.sdwa(sel=5") && - Common::ContainsStr(result.ir_dump, "v2.sdwa(sel=5,sext=0).abs"), - "V_CVT_F32_F16 SDWA source selector modifier did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ReadFirstLaneU32 s24, v5"), - "V_READFIRSTLANE_B32 did not lower to subgroup IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertFloorF32ToI32 v64, v6"), - "V_CVT_FLR_I32_F32 did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertI4ToOffsetF32 v81, v5"), - "V_CVT_OFF_F32_I4 did not lower to shared offset-convert IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v65, v5, 0x00000000"), - "V_CVT_F32_UBYTE0 did not lower to shared byte-convert IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v66, v5, 0x00000001"), - "V_CVT_F32_UBYTE1 did not lower to shared byte-convert IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v67, v5, 0x00000002"), - "V_CVT_F32_UBYTE2 did not lower to shared byte-convert IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertByteU32ToF32 v68, v5, 0x00000003"), - "V_CVT_F32_UBYTE3 did not lower to shared byte-convert IR"); - Check(Common::ContainsStr(result.ir_dump, "RcpF32 v11"), "VOP1 reciprocal did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FractF32 v12"), "VOP1 fract did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "TruncF32 v13"), "VOP1 trunc did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CeilF32 v14"), "VOP1 ceil did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "RoundEvenF32 v15"), - "VOP1 round-even did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FloorF32 v16"), "VOP1 floor did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "Exp2F32 v17"), "VOP1 exp2 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "Log2F32 v18"), "VOP1 log2 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "InverseSqrtF32 v19"), - "VOP1 inverse-sqrt did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "SqrtF32 v20"), "VOP1 sqrt did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "SinF32 v21"), "VOP1 sin did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CosF32 v22"), "VOP1 cos did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 v27"), "VOP1 not did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 v28"), - "VOP1 bit reverse did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU32 v31"), - "VOP1 find-first-bit-high did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FindLsbU32 v32"), - "VOP1 find-first-bit-low did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 v83, v5, v6"), - "V_XNOR_B32 did not lower to shared xnor IR"); - Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountLowU32 v84, v5, v6"), - "V_MBCNT_LO_U32_B32 did not lower to shared masked bit-count IR"); - Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountHighU32 v85, v5, v6"), - "V_MBCNT_HI_U32_B32 did not lower to shared masked bit-count IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v89, v6, v6, v89"), - "V_MAC_F32 did not lower with the destination register as addend"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v90, v6, 0x3f800000, v5"), - "V_MADMK_F32 did not lower with the literal in source 1"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v91, v6, v5, 0x40000000"), - "V_MADAK_F32 did not lower with the literal in source 2"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v92, v6, v6, v92"), - "alternate V_MAC_F32 did not lower with the destination register as addend"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v93, v6, 0x3f000000, v5"), - "alternate V_MADMK_F32 did not lower with the literal in source 1"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v94, v6, v5, 0x40400000"), - "alternate V_MADAK_F32 did not lower with the literal in source 2"); - Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v95, v6, v6"), - "native V_CVT_PKRTZ_F16_F32 did not lower to shared pack IR"); - Check(Common::ContainsStr(result.ir_dump, "Dot2AccF32F16 v102, v95, v95, v102"), - "V_DOT2C_F32_F16 did not lower to explicit dot-accumulate IR"); - Check(Common::ContainsStr(result.ir_dump, "IAddCarryU32 v97, vcc_lo, v5, v6, vcc_lo"), - "V_ADD_CO_U32 did not lower to carry-add IR"); - Check(Common::ContainsStr(result.ir_dump, "FAddF32 v35"), - "VOP3-encoded VOP2 float add did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "UMulU24U32 v36"), - "VOP3-encoded VOP2 24-bit multiply did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 v86, v5, v6"), - "VOP3-encoded V_XNOR_B32 did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v96, v6, v6, v96"), - "VOP3-encoded V_MAC_F32 did not lower with the destination register as addend"); - Check(Common::ContainsStr(result.ir_dump, "IAddCarryU32 v98, s30, v5, v6, vcc_lo"), - "VOP3-encoded V_ADD_CO_U32 did not lower to carry-add IR"); - Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountLowU32 v87, v5, v6"), - "VOP3-encoded V_MBCNT_LO_U32_B32 did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountHighU32 v88, v5, v6"), - "VOP3-encoded V_MBCNT_HI_U32_B32 did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 v23"), - "VOP3-encoded VOP1 move did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "ReadFirstLaneU32 s25, v5"), - "VOP3-encoded V_READFIRSTLANE_B32 did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "ReadLaneU32 s26, v5, 0x00000002"), - "V_READLANE_B32 did not lower to subgroup lane IR"); - Check(Common::ContainsStr(result.ir_dump, "WriteLaneU32 v107, v5, 0x00000002"), - "V_WRITELANE_B32 did not lower to subgroup lane IR"); - Check(Common::ContainsStr(result.ir_dump, "Permlane16B32 v108, v5, 0x00000000, 0x00000000"), - "V_PERMLANE16_B32 did not lower to subgroup perm-lane IR"); - Check(Common::ContainsStr(result.ir_dump, "Permlanex16B32 v109, v5, 0x00000000, 0x00000000"), - "V_PERMLANEX16_B32 did not lower to subgroup perm-lane IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertI32ToF32 v24"), - "VOP3-encoded VOP1 signed conversion did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertFloorF32ToI32 v74, v6"), - "VOP3-encoded V_CVT_FLR_I32_F32 did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertI4ToOffsetF32 v82, v5"), - "VOP3-encoded V_CVT_OFF_F32_I4 did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "FractF32 v25"), - "VOP3-encoded VOP1 fract did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "RcpF32 v26"), - "VOP3-encoded VOP1 reciprocal did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 v29"), - "VOP3-encoded VOP1 not did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 v30"), - "VOP3-encoded VOP1 bit reverse did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU32 v33"), - "VOP3-encoded VOP1 find-first-bit-high did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "FindLsbU32 v34"), - "VOP3-encoded VOP1 find-first-bit-low did not lower through shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareGtF32"), - "VOPC float compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtF32 exec_lo"), - "VOPC float compare-and-mask did not lower to exec mask IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtU32 exec_lo"), - "VOPC uint compare-and-mask did not lower to exec mask IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareFalse vcc_lo, v6, v6"), - "VOPC false compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareTrue vcc_lo, v6, v6"), - "VOPC true compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareOrderedF32 vcc_lo, v6, v6"), - "VOPC ordered compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnorderedF32 vcc_lo, v6, v6"), - "VOPC unordered compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordLtF32 vcc_lo, v6, v6"), - "VOPC unordered-less compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordEqF32 vcc_lo, v6, v6"), - "VOPC unordered-equal compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordLeF32 vcc_lo, v6, v6"), - "VOPC unordered-less-equal compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordGtF32 vcc_lo, v6, v6"), - "VOPC unordered-greater compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordNeF32 vcc_lo, v6, v6"), - "VOPC unordered-not-equal compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordNeF32 s0, 0x3f000000, v1"), - "VOPC SDWA unordered-not-equal compare did not lower to scalar-destination IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareUnordGeF32 vcc_lo, v6, v6"), - "VOPC unordered-greater-equal compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordLtF32 exec_lo, v6, v6"), - "VOPC unordered-less compare-and-mask did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordEqF32 exec_lo, v6, v6"), - "VOPC unordered-equal compare-and-mask did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordLeF32 exec_lo, v6, v6"), - "VOPC unordered-less-equal compare-and-mask did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordGtF32 exec_lo, v6, v6"), - "VOPC unordered-greater compare-and-mask did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordNeF32 exec_lo, v6, v6"), - "VOPC unordered-not-equal compare-and-mask did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskUnordGeF32 exec_lo, v6, v6"), - "VOPC unordered-greater-equal compare-and-mask did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareFalse vcc_lo, v5, v5"), - "VOPC integer false compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareTrue vcc_lo, v5, v5"), - "VOPC integer true compare did not lower to shared IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v8"), "VOP3 mad did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v4, v1.neg, v9, s4"), - "VOP3 mad source modifiers did not lower to IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "CubeIdF32 v110, v5, v6, v7"), - "V_CUBEID_F32 did not lower to cube IR"); - Check(Common::ContainsStr(result.ir_dump, "CubeScF32 v111, v5, v6, v7"), - "V_CUBESC_F32 did not lower to cube IR"); - Check(Common::ContainsStr(result.ir_dump, "CubeTcF32 v112, v5, v6, v7"), - "V_CUBETC_F32 did not lower to cube IR"); - Check(Common::ContainsStr(result.ir_dump, "CubeMaF32 v113, v5, v6, v7"), - "V_CUBEMA_F32 did not lower to cube IR"); - Check(Common::ContainsStr(result.ir_dump, "CubeMaF32 v5, v2, v0, v6.neg"), - "V_CUBEMA_F32 source modifier did not lower to cube IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v37"), "VOP3 fma did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FMin3F32 v38"), "VOP3 min3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FMax3F32 v39"), "VOP3 max3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FMed3F32 v40"), "VOP3 med3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMin3I32 v41"), - "VOP3 signed min3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "UMin3U32 v42"), - "VOP3 unsigned min3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMax3I32 v43"), - "VOP3 signed max3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "UMax3U32 v44"), - "VOP3 unsigned max3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMed3I32 v45"), - "VOP3 signed med3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "UMed3U32 v46"), - "VOP3 unsigned med3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldExtract3U32 v47"), - "VOP3 unsigned bitfield extract did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldExtract3I32 v48"), - "VOP3 signed bitfield extract did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldInsertSelectU32 v49"), - "VOP3 bitfield insert-select did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AlignBitU32 v50"), - "VOP3 alignbit did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IAdd3U32 v51"), "VOP3 add3 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMulU32 v52, v5, v6"), - "V_MUL_LO_U32 did not lower to multiply IR"); - Check(Common::ContainsStr(result.ir_dump, "UMulHighU32 v53, v5, v6"), - "V_MUL_HI_U32 did not lower to high-multiply IR"); - Check(Common::ContainsStr(result.ir_dump, "SMulHighI32 v114, v5, v6"), - "V_MUL_HI_I32 did not lower to signed high-multiply IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndOrU32 v54, v5, v6, v7"), - "V_AND_OR_B32 did not lower to ternary bitwise IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseOr3U32 v55, v5, v6, v7"), - "V_OR3_B32 did not lower to ternary bitwise IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseXor3U32 v56, v5, v6, v7"), - "V_XOR3_B32 did not lower to ternary bitwise IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftLeftAddU32 v57, v5, v6, v7"), - "V_LSHL_ADD_U32 did not lower to shared shift-left-add IR"); - Check(Common::ContainsStr(result.ir_dump, "AddShiftLeftU32 v58, v5, v6, v7"), - "V_ADD_LSHL_U32 did not lower to add-shift-left IR"); - Check(Common::ContainsStr(result.ir_dump, "XorAddU32 v59, v5, v6, v7"), - "V_XAD_U32 did not lower to xor-add IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftLeftOrU32 v60, v5, v6, v7"), - "V_LSHL_OR_B32 did not lower to shift-left-or IR"); - Check(Common::ContainsStr(result.ir_dump, "SadU32 v61, v5, v6, v7"), - "V_SAD_U32 did not lower to sad IR"); - Check(Common::ContainsStr(result.ir_dump, "IMadI24U32 v62, v5, v6, v7"), - "V_MAD_I32_I24 did not lower to signed 24-bit mad IR"); - Check(Common::ContainsStr(result.ir_dump, "UMadU24U32 v63, v5, v6, v7"), - "V_MAD_U32_U24 did not lower to unsigned 24-bit mad IR"); - Check(Common::ContainsStr(result.ir_dump, "IMulU32 v69, v5, v6"), - "V_MUL_LO_I32 did not lower to multiply IR"); - Check(Common::ContainsStr(result.ir_dump, "IAddCarryU32 v70, s0, v5, v6, 0x00000000"), - "V_ADD_I32 did not lower to carry-out add IR"); - Check(Common::ContainsStr(result.ir_dump, "ISubBorrowU32 v71, s0, v5, v6"), - "V_SUB_I32 did not lower to borrow-out subtract IR"); - Check(Common::ContainsStr(result.ir_dump, "ISubBorrowU32 v72, s0, v6, v5"), - "V_SUBREV_I32 did not reverse source order in borrow-out IR"); - Check(Common::ContainsStr( - result.ir_dump, - "IMaxI16 v7.sdwa(sel=4,sext=0), 0x0000ffff, v5.opsel(lo=1,hi=0,neghi=0)"), - "V_MAX_I16 did not lower to signed halfword max IR"); - Check(Common::ContainsStr( - result.ir_dump, - "IMinI16 v7.sdwa(sel=5,sext=0), 0x00000002, v4.opsel(lo=1,hi=0,neghi=0)"), - "V_MIN_I16 did not lower to signed halfword min IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU32 v73, 0x00000001, 0x00000004"), - "V_BFM_B32 did not lower to shared bitfield-mask IR"); - Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v75, v6, v6"), - "V_CVT_PKRTZ_F16_F32 did not lower to shared pack IR"); - Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v0, v0.neg, v1.neg"), - "V_CVT_PKRTZ_F16_F32 source modifiers did not lower to shared pack IR"); - Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v76, v6, 0x00000001"), - "V_LDEXP_F32 did not lower to ldexp IR"); - Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v7, v9.abs, 0xfffffffe"), - "V_LDEXP_F32 source modifier did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v13.clamp, v6, 0xfffffffc"), - "V_LDEXP_F32 clamp modifier did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitCountAddU32 v77, v5, v6"), - "VOP3 V_BCNT_U32_B32 did not lower to bit-count-add IR"); - Check(Common::ContainsStr(result.ir_dump, "PackSnorm2x16F32 v78, v6, v6"), - "V_CVT_PKNORM_I16_F32 did not lower to shared pack IR"); - Check(Common::ContainsStr(result.ir_dump, "PackUnorm2x16F32 v79, v6, v6"), - "V_CVT_PKNORM_U16_F32 did not lower to shared pack IR"); - Check(Common::ContainsStr(result.ir_dump, "PackU16U32 v80, v5, v7"), - "V_CVT_PK_U16_U32 did not lower to shared pack IR"); - Check(SpirvContainsOpcode(result.spirv, 112), "SPIR-V binary does not contain OpConvertUToF"); - Check(SpirvContainsOpcode(result.spirv, 109), "SPIR-V binary does not contain OpConvertFToU"); - Check(SpirvContainsOpcode(result.spirv, 111), "SPIR-V binary does not contain OpConvertSToF"); - Check(SpirvContainsOpcode(result.spirv, 110), "SPIR-V binary does not contain OpConvertFToS"); - Check(SpirvContainsOpcode(result.spirv, 136), "SPIR-V binary does not contain OpFDiv"); - Check(SpirvContainsOpcode(result.spirv, 127), "SPIR-V binary does not contain OpFNegate"); - Check(SpirvContainsOpcode(result.spirv, 12), "SPIR-V binary does not contain OpExtInst"); - Check(SpirvContainsExtInst(result.spirv, 58), - "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16"); - Check(SpirvContainsExtInst(result.spirv, 62), - "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16"); - Check(SpirvContainsExtInst(result.spirv, 50), - "SPIR-V binary does not contain GLSL.std.450 Fma"); - Check(SpirvContainsExtInst(result.spirv, 43), - "SPIR-V binary does not contain GLSL.std.450 FClamp"); - Check(SpirvContainsExtInst(result.spirv, 53), - "SPIR-V binary does not contain GLSL.std.450 Ldexp"); - Check(SpirvContainsOpcode(result.spirv, 128), "SPIR-V binary does not contain OpIAdd"); - Check(SpirvContainsOpcode(result.spirv, 149), "SPIR-V binary does not contain OpIAddCarry"); - Check(SpirvContainsOpcode(result.spirv, 130), "SPIR-V binary does not contain OpISub"); - Check(SpirvContainsOpcode(result.spirv, 132), "SPIR-V binary does not contain OpIMul"); - Check(SpirvContainsOpcode(result.spirv, 171), "SPIR-V binary does not contain OpINotEqual"); - Check(SpirvContainsOpcode(result.spirv, 200), "SPIR-V binary does not contain OpNot"); - Check(SpirvContainsOpcode(result.spirv, 204), "SPIR-V binary does not contain OpBitReverse"); - Check(SpirvContainsOpcode(result.spirv, 205), "SPIR-V binary does not contain OpBitCount"); - Check(SpirvContainsOpcode(result.spirv, 166), "SPIR-V binary does not contain OpLogicalOr"); - Check(SpirvContainsOpcode(result.spirv, 167), "SPIR-V binary does not contain OpLogicalAnd"); - Check(SpirvContainsOpcode(result.spirv, 186), - "SPIR-V binary does not contain OpFOrdGreaterThan"); - Check(SpirvContainsOpcode(result.spirv, 181), "SPIR-V binary does not contain OpFUnordEqual"); - Check(SpirvContainsOpcode(result.spirv, 183), - "SPIR-V binary does not contain OpFUnordNotEqual"); - Check(SpirvContainsOpcode(result.spirv, 185), - "SPIR-V binary does not contain OpFUnordLessThan"); - Check(SpirvContainsOpcode(result.spirv, 187), - "SPIR-V binary does not contain OpFUnordGreaterThan"); - Check(SpirvContainsOpcode(result.spirv, 189), - "SPIR-V binary does not contain OpFUnordLessThanEqual"); - Check(SpirvContainsOpcode(result.spirv, 191), - "SPIR-V binary does not contain OpFUnordGreaterThanEqual"); - Check(SpirvContainsOpcode(result.spirv, 190), - "SPIR-V binary does not contain OpFOrdGreaterThanEqual"); - Check(SpirvContainsOpcode(result.spirv, 184), "SPIR-V binary does not contain OpFOrdLessThan"); - Check(SpirvContainsOpcode(result.spirv, 173), "SPIR-V binary does not contain OpSGreaterThan"); - Check(SpirvContainsOpcode(result.spirv, 174), - "SPIR-V binary does not contain OpUGreaterThanEqual"); - Check(SpirvContainsOpcode(result.spirv, 177), "SPIR-V binary does not contain OpSLessThan"); - Check(SpirvContainsOpcode(result.spirv, 202), - "SPIR-V binary does not contain OpBitFieldSExtract"); - Check(SpirvContainsOpcode(result.spirv, 203), - "SPIR-V binary does not contain OpBitFieldUExtract"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad for cmpx old exec mask"); - Check(SpirvContainsOpcode(result.spirv, 133), "SPIR-V binary does not contain OpFMul"); - Check(SpirvContainsOpcode(result.spirv, 129), "SPIR-V binary does not contain OpFAdd"); - Check(SpirvContainsOpcode(result.spirv, 151), "SPIR-V binary does not contain OpUMulExtended"); - Check(SpirvContainsOpcode(result.spirv, 152), "SPIR-V binary does not contain OpSMulExtended"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 197), "SPIR-V binary does not contain OpBitwiseOr"); - Check(SpirvContainsOpcode(result.spirv, 198), "SPIR-V binary does not contain OpBitwiseXor"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - Check(SpirvContainsOpcode(result.spirv, 80), - "SPIR-V binary does not contain OpCompositeConstruct"); - Check(SpirvContainsOpcode(result.spirv, 81), - "SPIR-V binary does not contain OpCompositeExtract"); - Check(SpirvContainsOpcode(result.spirv, 339), - "SPIR-V binary does not contain OpGroupNonUniformBallot"); - Check(SpirvContainsOpcode(result.spirv, 343), - "SPIR-V binary does not contain OpGroupNonUniformBallotFindLSB"); - Check(SpirvContainsOpcode(result.spirv, 345), - "SPIR-V binary does not contain OpGroupNonUniformShuffle"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_movk_i32 s9"), + "new decoder did not decode SOPK mov"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32"), + "new decoder did not decode SOPK compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_nop"), + "new decoder did not decode old-backed VOP1 no-op"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v5"), + "new decoder did not decode VOP1 mov"); + Check(Common::ContainsStr(result.decoded_dump, "v_movrels_b32 v55, v12"), + "new decoder did not decode VOP1 V_MOVRELS_B32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_mov_b32 v103, v5.sdwa(sel=4"), + "new decoder did not decode VOP1 SDWA source selector"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v105.sdwa(sel=5"), + "new decoder did not decode VOP1 SDWA destination selector"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v106, v5.dpp"), + "new decoder did not decode VOP1 DPP source metadata"); + Check(!Common::ContainsStr(result.decoded_dump, + "VOP1 SDWA/DPP modifiers are not implemented"), + "new decoder still reports blanket VOP1 SDWA/DPP unsupported reason"); + Check(Common::ContainsStr(result.decoded_dump, + "v_add_nc_u32 v123, v5.sdwa(sel=4"), + "new decoder did not decode VOP2 SDWA source selector"); + Check( + Common::ContainsStr(result.decoded_dump, "v_and_b32 v124, v5.sdwa(sel=4"), + "new decoder did not decode VOP2 SDWA first source selector"); + Check(Common::ContainsStr(result.decoded_dump, "v6.sdwa(sel=5"), + "new decoder did not decode VOP2 SDWA second source selector"); + Check(Common::ContainsStr(result.decoded_dump, "v_cndmask_b32 v47, v53,") && + Common::ContainsStr(result.decoded_dump, "v53.neg"), + "new decoder did not decode V_CNDMASK_B32 SDWA source modifier"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cndmask_b32 v7, v0.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v1"), + "new decoder did not decode full-destination V_CNDMASK_B32 with SDWA " + "source"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v125, v5.dpp"), + "new decoder did not decode VOP2 DPP source metadata"); + Check(!Common::ContainsStr(result.decoded_dump, + "VOP2 SDWA/DPP modifiers are not implemented"), + "new decoder still reports blanket VOP2 SDWA/DPP unsupported reason"); + Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s24, v5"), + "new decoder did not decode old-backed V_READFIRSTLANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_u32 v6"), + "new decoder did not decode VOP1 conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_i32 v9"), + "new decoder did not decode VOP1 signed int-to-float conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_i32_f32 v10"), + "new decoder did not decode VOP1 float-to-signed-int conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v99"), + "new decoder did not decode old-backed V_CVT_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_f16 v100"), + "new decoder did not decode old-backed native V_CVT_F32_F16"); + Check( + Common::ContainsStr(result.decoded_dump, + "v_cvt_f32_f16 v8, v2.sdwa(sel=5") && + Common::ContainsStr(result.decoded_dump, "v2.sdwa(sel=5,sext=0).abs"), + "new decoder did not decode V_CVT_F32_F16 SDWA source selector modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_flr_i32_f32 v64"), + "new decoder did not decode old-backed V_CVT_FLR_I32_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_off_f32_i4 v81"), + "new decoder did not decode old-backed V_CVT_OFF_F32_I4"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte0 v65"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE0"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte1 v66"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE1"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte2 v67"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE2"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_ubyte3 v68"), + "new decoder did not decode old-backed V_CVT_F32_UBYTE3"); + Check(Common::ContainsStr(result.decoded_dump, "v_rcp_f32 v11"), + "new decoder did not decode VOP1 reciprocal"); + Check(Common::ContainsStr(result.decoded_dump, "v_fract_f32 v12"), + "new decoder did not decode VOP1 fract"); + Check(Common::ContainsStr(result.decoded_dump, "v_cos_f32 v22"), + "new decoder did not decode VOP1 cosine"); + Check(Common::ContainsStr(result.decoded_dump, "v_not_b32 v27"), + "new decoder did not decode VOP1 not"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfrev_b32 v28"), + "new decoder did not decode VOP1 bit reverse"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbh_u32 v31"), + "new decoder did not decode VOP1 find-first-bit-high"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbl_b32 v32"), + "new decoder did not decode VOP1 find-first-bit-low"); + Check(Common::ContainsStr(result.decoded_dump, "v_xnor_b32 v83, v5, v6"), + "new decoder did not decode old-backed V_XNOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_mbcnt_lo_u32_b32 v84, v5, v6"), + "new decoder did not decode old-backed V_MBCNT_LO_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_mbcnt_hi_u32_b32 v85, v5, v6"), + "new decoder did not decode old-backed V_MBCNT_HI_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v89, v6, v6"), + "new decoder did not decode old-backed V_MAC_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_madmk_f32 v90, v6, 0x3f800000, v5"), + "new decoder did not decode old-backed V_MADMK_F32 literal form"); + Check(Common::ContainsStr(result.decoded_dump, + "v_madak_f32 v91, v6, v5, 0x40000000"), + "new decoder did not decode old-backed V_MADAK_F32 literal form"); + Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v92, v6, v6"), + "new decoder did not decode old-backed alternate V_MAC_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_madmk_f32 v93, v6, 0x3f000000, v5"), + "new decoder did not decode old-backed alternate V_MADMK_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_madak_f32 v94, v6, v5, 0x40400000"), + "new decoder did not decode old-backed alternate V_MADAK_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cvt_pkrtz_f16_f32 v95, v6, v6"), + "new decoder did not decode old-backed native V_CVT_PKRTZ_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_dot2c_f32_f16 v102, v95, v95"), + "new decoder did not decode old-backed V_DOT2C_F32_F16"); + Check(Common::ContainsStr(result.decoded_dump, + "v_addc_u32 v97, vcc_lo, v5, v6, vcc_lo"), + "new decoder did not decode old-backed V_ADD_CO_U32 carry form"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_f32 v35"), + "new decoder did not decode VOP3-encoded VOP2 float add"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_u32_u24 v36"), + "new decoder did not decode VOP3-encoded VOP2 24-bit multiply"); + Check(Common::ContainsStr(result.decoded_dump, "v_xnor_b32 v86, v5, v6"), + "new decoder did not decode old-backed VOP3-encoded V_XNOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mac_f32 v96, v6, v6"), + "new decoder did not decode old-backed VOP3-encoded V_MAC_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_addc_u32 v98, s30, v5, v6, vcc_lo"), + "new decoder did not decode old-backed VOP3 V_ADD_CO_U32 carry form"); + Check( + Common::ContainsStr(result.decoded_dump, + "v_mbcnt_lo_u32_b32 v87, v5, v6"), + "new decoder did not decode old-backed VOP3-encoded V_MBCNT_LO_U32_B32"); + Check( + Common::ContainsStr(result.decoded_dump, + "v_mbcnt_hi_u32_b32 v88, v5, v6"), + "new decoder did not decode old-backed VOP3-encoded V_MBCNT_HI_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v23"), + "new decoder did not decode VOP3-encoded VOP1 move"); + Check( + Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), + "new decoder did not decode old-backed VOP3-encoded V_READFIRSTLANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f32_i32 v24"), + "new decoder did not decode VOP3-encoded VOP1 signed conversion"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_f32 v101"), + "new decoder did not decode old-backed VOP3-encoded V_CVT_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_flr_i32_f32 v74"), + "new decoder did not decode old-backed VOP3-encoded V_CVT_FLR_I32_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_off_f32_i4 v82"), + "new decoder did not decode old-backed VOP3-encoded V_CVT_OFF_F32_I4"); + Check(Common::ContainsStr(result.decoded_dump, "v_fract_f32 v25"), + "new decoder did not decode VOP3-encoded VOP1 fract"); + Check(Common::ContainsStr(result.decoded_dump, "v_rcp_f32 v26"), + "new decoder did not decode VOP3-encoded VOP1 reciprocal"); + Check(Common::ContainsStr(result.decoded_dump, "v_not_b32 v29"), + "new decoder did not decode VOP3-encoded VOP1 not"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfrev_b32 v30"), + "new decoder did not decode VOP3-encoded VOP1 bit reverse"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbh_u32 v33"), + "new decoder did not decode VOP3-encoded VOP1 find-first-bit-high"); + Check(Common::ContainsStr(result.decoded_dump, "v_ffbl_b32 v34"), + "new decoder did not decode VOP3-encoded VOP1 find-first-bit-low"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_gt_f32"), + "new decoder did not decode VOPC float compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_f32"), + "new decoder did not decode VOPC float compare-and-mask"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_u32"), + "new decoder did not decode VOPC uint compare-and-mask"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_f32"), + "new decoder did not decode old-backed V_CMP_F_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_tru_f32"), + "new decoder did not decode old-backed V_CMP_TRU_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_o_f32"), + "new decoder did not decode old-backed V_CMP_O_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_u_f32"), + "new decoder did not decode old-backed V_CMP_U_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nge_f32"), + "new decoder did not decode old-backed V_CMP_NGE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nlg_f32"), + "new decoder did not decode old-backed V_CMP_NLG_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ngt_f32"), + "new decoder did not decode old-backed V_CMP_NGT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nle_f32"), + "new decoder did not decode old-backed V_CMP_NLE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_neq_f32"), + "new decoder did not decode old-backed V_CMP_NEQ_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cmp_neq_f32 s0, 0.500000, v1"), + "new decoder did not decode old-backed V_CMP_NEQ_F32 SDWA scalar " + "destination"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cmpx_lt_u32 exec_lo, v5.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "CompareMaskLtU32 exec_lo"), + "new decoder did not route V_CMPX SDWA destination to exec"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_nlt_f32"), + "new decoder did not decode old-backed V_CMP_NLT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nge_f32"), + "new decoder did not decode old-backed V_CMPX_NGE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nlg_f32"), + "new decoder did not decode old-backed V_CMPX_NLG_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_ngt_f32"), + "new decoder did not decode old-backed V_CMPX_NGT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nle_f32"), + "new decoder did not decode old-backed V_CMPX_NLE_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_neq_f32"), + "new decoder did not decode old-backed V_CMPX_NEQ_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_nlt_f32"), + "new decoder did not decode old-backed V_CMPX_NLT_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_i32"), + "new decoder did not decode old-backed V_CMP_F_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_t_i32"), + "new decoder did not decode old-backed V_CMP_T_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_f_u32"), + "new decoder did not decode old-backed V_CMP_F_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_t_u32"), + "new decoder did not decode old-backed V_CMP_T_U32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cmp_ne_u64 vcc_lo, exec_lo, vcc_lo"), + "new decoder did not decode VOP3-encoded V_CMP_NE_U64"); + Check(Common::ContainsStr(result.ir_dump, + "CompareNeU64 vcc_lo, exec_lo, vcc_lo"), + "V_CMP_NE_U64 did not lower to 64-bit compare IR"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_f32 v8"), + "new decoder did not decode VOP3 mad"); + Check( + Common::ContainsStr(result.decoded_dump, "v_mad_f32 v4, v1.neg, v9, s4"), + "new decoder did not decode VOP3 mad source modifiers"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cubeid_f32 v110, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBEID_F32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cubesc_f32 v111, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBESC_F32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cubetc_f32 v112, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBETC_F32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cubema_f32 v113, v5, v6, v7"), + "new decoder did not decode old-backed V_CUBEMA_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cubema_f32 v5, v2, v0, v6.neg"), + "new decoder did not decode V_CUBEMA_F32 source modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v37"), + "new decoder did not decode VOP3 fma"); + Check(Common::ContainsStr(result.decoded_dump, "v_min3_f32 v38"), + "new decoder did not decode VOP3 min3"); + Check(Common::ContainsStr(result.decoded_dump, "v_max3_f32 v39"), + "new decoder did not decode VOP3 max3"); + Check(Common::ContainsStr(result.decoded_dump, "v_med3_f32 v40"), + "new decoder did not decode VOP3 med3"); + Check(Common::ContainsStr(result.decoded_dump, "v_min3_i32 v41"), + "new decoder did not decode VOP3 signed min3"); + Check(Common::ContainsStr(result.decoded_dump, "v_min3_u32 v42"), + "new decoder did not decode VOP3 unsigned min3"); + Check(Common::ContainsStr(result.decoded_dump, "v_max3_i32 v43"), + "new decoder did not decode VOP3 signed max3"); + Check(Common::ContainsStr(result.decoded_dump, "v_max3_u32 v44"), + "new decoder did not decode VOP3 unsigned max3"); + Check(Common::ContainsStr(result.decoded_dump, "v_med3_i32 v45"), + "new decoder did not decode VOP3 signed med3"); + Check(Common::ContainsStr(result.decoded_dump, "v_med3_u32 v46"), + "new decoder did not decode VOP3 unsigned med3"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfe_u32 v47"), + "new decoder did not decode VOP3 unsigned bitfield extract"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfe_i32 v48"), + "new decoder did not decode VOP3 signed bitfield extract"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfi_b32 v49"), + "new decoder did not decode VOP3 bitfield insert-select"); + Check(Common::ContainsStr(result.decoded_dump, "v_alignbit_b32 v50"), + "new decoder did not decode VOP3 alignbit"); + Check(Common::ContainsStr(result.decoded_dump, "v_add3_u32 v51"), + "new decoder did not decode VOP3 add3"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_lo_u32 v52, v5, v6"), + "new decoder did not decode old-backed V_MUL_LO_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_hi_u32 v53, v5, v6"), + "new decoder did not decode old-backed V_MUL_HI_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_hi_i32 v114, v5, v6"), + "new decoder did not decode RDNA2 V_MUL_HI_I32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_and_or_b32 v54, v5, v6, v7"), + "new decoder did not decode old-backed V_AND_OR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_or3_b32 v55, v5, v6, v7"), + "new decoder did not decode old-backed V_OR3_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_xor3_b32 v56, v5, v6, v7"), + "new decoder did not decode old-backed V_XOR3_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_lshl_add_u32 v57, v5, v6, v7"), + "new decoder did not decode old-backed V_LSHL_ADD_U32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_add_lshl_u32 v58, v5, v6, v7"), + "new decoder did not decode old-backed V_ADD_LSHL_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_xad_u32 v59, v5, v6, v7"), + "new decoder did not decode old-backed V_XAD_U32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_lshl_or_b32 v60, v5, v6, v7"), + "new decoder did not decode old-backed V_LSHL_OR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_sad_u32 v61, v5, v6, v7"), + "new decoder did not decode old-backed V_SAD_U32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_mad_i32_i24 v62, v5, v6, v7"), + "new decoder did not decode old-backed V_MAD_I32_I24"); + Check( + Common::ContainsStr(result.decoded_dump, "v_mad_u32_u24 v63, v5, v6, v7"), + "new decoder did not decode old-backed V_MAD_U32_U24"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_lo_i32 v69, v5, v6"), + "new decoder did not decode old-backed V_MUL_LO_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_i32 v70, s0, v5, v6"), + "new decoder did not decode old-backed V_ADD_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_i32 v71, s0, v5, v6"), + "new decoder did not decode RDNA2 V_SUB_CO_U32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_subrev_i32 v72, s0, v5, v6"), + "new decoder did not decode old-backed V_SUBREV_I32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_max_i16 v7.sdwa(sel=4,sext=0), 0x0000ffff, " + "v5.opsel(lo=1,hi=0,neghi=0)"), + "new decoder did not decode RDNA2 V_MAX_I16 literal/op_sel form"); + Check(Common::ContainsStr( + result.decoded_dump, + "v_min_i16 v7.sdwa(sel=5,sext=0), 2, v4.opsel(lo=1,hi=0,neghi=0)"), + "new decoder did not decode RDNA2 V_MIN_I16 op_sel form"); + Check(Common::ContainsStr(result.decoded_dump, "v_bfm_b32 v73, 1, 4"), + "new decoder did not decode old-backed V_BFM_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), + "new decoder did not decode old-backed V_READLANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_writelane_b32 v107, v5, 2"), + "new decoder did not decode old-backed V_WRITELANE_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_permlane16_b32 v108, v5, 0, 0"), + "new decoder did not decode old-backed V_PERMLANE16_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_permlanex16_b32 v109, v5, 0, 0"), + "new decoder did not decode old-backed V_PERMLANEX16_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cvt_pkrtz_f16_f32 v75, v6, v6"), + "new decoder did not decode old-backed V_CVT_PKRTZ_F16_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cvt_pkrtz_f16_f32 v0, v0.neg, v1.neg"), + "new decoder did not decode V_CVT_PKRTZ_F16_F32 source modifiers"); + Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v76, v6, 1"), + "new decoder did not decode old-backed V_LDEXP_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v7, v9.abs, -2"), + "new decoder did not decode V_LDEXP_F32 source modifier"); + Check( + Common::ContainsStr(result.decoded_dump, "v_ldexp_f32 v13.clamp, v6, -4"), + "new decoder did not decode V_LDEXP_F32 clamp modifier"); + Check(Common::ContainsStr(result.decoded_dump, "v_bcnt_u32_b32 v77, v5, v6"), + "new decoder did not decode old-backed VOP3 V_BCNT_U32_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cvt_pknorm_i16_f32 v78, v6, v6"), + "new decoder did not decode old-backed V_CVT_PKNORM_I16_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_cvt_pknorm_u16_f32 v79, v6, v6"), + "new decoder did not decode old-backed V_CVT_PKNORM_U16_F32"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cvt_pk_u16_u32 v80, v5, v7"), + "new decoder did not decode old-backed V_CVT_PK_U16_U32"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_f32 v2, s0, s72"), + "new decoder did not decode old-backed V_MUL_F32 SDWA full-width form"); + Check(Common::ContainsStr(result.decoded_dump, "v_mul_f16 v9.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v0.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v1.sdwa(sel=4"), + "new decoder did not decode V_MUL_F16 SDWA low-half form"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_f16 v0.sdwa(sel=4") && + Common::ContainsStr(result.decoded_dump, "v0.sdwa(sel=4"), + "new decoder did not decode V_SUB_F16 SDWA low-half form"); + Check(Common::ContainsStr(result.ir_dump, "ConvertU32ToF32 v6"), + "VOP1 uint-to-float conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v103, v5.sdwa(sel=4"), + "VOP1 SDWA source selector did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToF16 v105.sdwa(sel=5"), + "VOP1 SDWA destination selector did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v106.dpp") && + Common::ContainsStr(result.ir_dump, "v5.dpp"), + "VOP1 DPP source/destination did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "MoveRelSourceU32 v55, v12, m0"), + "V_MOVRELS_B32 did not lower to indexed VGPR-source IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU32 v123, v5.sdwa(sel=4"), + "VOP2 SDWA did not lower first source metadata to IR"); + Check( + Common::ContainsStr(result.ir_dump, "BitwiseAndU32 v124, v5.sdwa(sel=4"), + "VOP2 SDWA bitwise op did not lower first source metadata to IR"); + Check(Common::ContainsStr(result.ir_dump, "v6.sdwa(sel=5"), + "VOP2 SDWA bitwise op did not lower second source metadata to IR"); + Check(Common::ContainsStr(result.ir_dump, + "SelectMaskF32Bits v47, vcc_lo, v53.neg, v53") && + Common::ContainsStr(result.ir_dump, "v53.neg"), + "V_CNDMASK_B32 SDWA source modifier did not lower to float-bit select " + "IR"); + Check(Common::ContainsStr(result.ir_dump, + "SelectMaskU32 v7, vcc_lo, v1, v0.sdwa(sel=4"), + "full-destination V_CNDMASK_B32 with SDWA source did not lower to " + "integer select IR"); + Check(Common::ContainsStr(result.ir_dump, "FAddF32 v125.dpp") && + Common::ContainsStr(result.ir_dump, "v5.dpp"), + "VOP2 DPP source/destination did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "FMulF32 v2, s0, s72"), + "V_MUL_F32 SDWA full-width form did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "MulF16 v9.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "v0.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "v1.sdwa(sel=4"), + "V_MUL_F16 SDWA low-half form did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SubF16 v0.sdwa(sel=4") && + Common::ContainsStr(result.ir_dump, "v0.sdwa(sel=4"), + "V_SUB_F16 SDWA low-half form did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToU32 v7"), + "VOP1 float-to-uint conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI32ToF32 v9"), + "VOP1 signed int-to-float conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToI32 v10"), + "VOP1 float-to-signed-int conversion did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF32ToF16 v99, v6"), + "V_CVT_F16_F32 did not lower to shared half-pack IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToF32 v100, v99"), + "V_CVT_F32_F16 did not lower to shared half-unpack IR"); + Check(Common::ContainsStr(result.ir_dump, + "ConvertF16ToF32 v8, v2.sdwa(sel=5") && + Common::ContainsStr(result.ir_dump, "v2.sdwa(sel=5,sext=0).abs"), + "V_CVT_F32_F16 SDWA source selector modifier did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ReadFirstLaneU32 s24, v5"), + "V_READFIRSTLANE_B32 did not lower to subgroup IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertFloorF32ToI32 v64, v6"), + "V_CVT_FLR_I32_F32 did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI4ToOffsetF32 v81, v5"), + "V_CVT_OFF_F32_I4 did not lower to shared offset-convert IR"); + Check(Common::ContainsStr(result.ir_dump, + "ConvertByteU32ToF32 v65, v5, 0x00000000"), + "V_CVT_F32_UBYTE0 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, + "ConvertByteU32ToF32 v66, v5, 0x00000001"), + "V_CVT_F32_UBYTE1 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, + "ConvertByteU32ToF32 v67, v5, 0x00000002"), + "V_CVT_F32_UBYTE2 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, + "ConvertByteU32ToF32 v68, v5, 0x00000003"), + "V_CVT_F32_UBYTE3 did not lower to shared byte-convert IR"); + Check(Common::ContainsStr(result.ir_dump, "RcpF32 v11"), + "VOP1 reciprocal did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FractF32 v12"), + "VOP1 fract did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "TruncF32 v13"), + "VOP1 trunc did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CeilF32 v14"), + "VOP1 ceil did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "RoundEvenF32 v15"), + "VOP1 round-even did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FloorF32 v16"), + "VOP1 floor did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "Exp2F32 v17"), + "VOP1 exp2 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "Log2F32 v18"), + "VOP1 log2 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "InverseSqrtF32 v19"), + "VOP1 inverse-sqrt did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SqrtF32 v20"), + "VOP1 sqrt did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SinF32 v21"), + "VOP1 sin did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CosF32 v22"), + "VOP1 cos did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 v27"), + "VOP1 not did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 v28"), + "VOP1 bit reverse did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU32 v31"), + "VOP1 find-first-bit-high did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FindLsbU32 v32"), + "VOP1 find-first-bit-low did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 v83, v5, v6"), + "V_XNOR_B32 did not lower to shared xnor IR"); + Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountLowU32 v84, v5, v6"), + "V_MBCNT_LO_U32_B32 did not lower to shared masked bit-count IR"); + Check( + Common::ContainsStr(result.ir_dump, "MaskedBitCountHighU32 v85, v5, v6"), + "V_MBCNT_HI_U32_B32 did not lower to shared masked bit-count IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v89, v6, v6, v89"), + "V_MAC_F32 did not lower with the destination register as addend"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v90, v6, 0x3f800000, v5"), + "V_MADMK_F32 did not lower with the literal in source 1"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v91, v6, v5, 0x40000000"), + "V_MADAK_F32 did not lower with the literal in source 2"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v92, v6, v6, v92"), + "alternate V_MAC_F32 did not lower with the destination register as " + "addend"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v93, v6, 0x3f000000, v5"), + "alternate V_MADMK_F32 did not lower with the literal in source 1"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v94, v6, v5, 0x40400000"), + "alternate V_MADAK_F32 did not lower with the literal in source 2"); + Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v95, v6, v6"), + "native V_CVT_PKRTZ_F16_F32 did not lower to shared pack IR"); + Check( + Common::ContainsStr(result.ir_dump, "Dot2AccF32F16 v102, v95, v95, v102"), + "V_DOT2C_F32_F16 did not lower to explicit dot-accumulate IR"); + Check(Common::ContainsStr(result.ir_dump, + "IAddCarryU32 v97, vcc_lo, v5, v6, vcc_lo"), + "V_ADD_CO_U32 did not lower to carry-add IR"); + Check(Common::ContainsStr(result.ir_dump, "FAddF32 v35"), + "VOP3-encoded VOP2 float add did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulU24U32 v36"), + "VOP3-encoded VOP2 24-bit multiply did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 v86, v5, v6"), + "VOP3-encoded V_XNOR_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v96, v6, v6, v96"), + "VOP3-encoded V_MAC_F32 did not lower with the destination register as " + "addend"); + Check(Common::ContainsStr(result.ir_dump, + "IAddCarryU32 v98, s30, v5, v6, vcc_lo"), + "VOP3-encoded V_ADD_CO_U32 did not lower to carry-add IR"); + Check(Common::ContainsStr(result.ir_dump, "MaskedBitCountLowU32 v87, v5, v6"), + "VOP3-encoded V_MBCNT_LO_U32_B32 did not lower through shared IR"); + Check( + Common::ContainsStr(result.ir_dump, "MaskedBitCountHighU32 v88, v5, v6"), + "VOP3-encoded V_MBCNT_HI_U32_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v23"), + "VOP3-encoded VOP1 move did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ReadFirstLaneU32 s25, v5"), + "VOP3-encoded V_READFIRSTLANE_B32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ReadLaneU32 s26, v5, 0x00000002"), + "V_READLANE_B32 did not lower to subgroup lane IR"); + Check( + Common::ContainsStr(result.ir_dump, "WriteLaneU32 v107, v5, 0x00000002"), + "V_WRITELANE_B32 did not lower to subgroup lane IR"); + Check(Common::ContainsStr(result.ir_dump, + "Permlane16B32 v108, v5, 0x00000000, 0x00000000"), + "V_PERMLANE16_B32 did not lower to subgroup perm-lane IR"); + Check(Common::ContainsStr(result.ir_dump, + "Permlanex16B32 v109, v5, 0x00000000, 0x00000000"), + "V_PERMLANEX16_B32 did not lower to subgroup perm-lane IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI32ToF32 v24"), + "VOP3-encoded VOP1 signed conversion did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertFloorF32ToI32 v74, v6"), + "VOP3-encoded V_CVT_FLR_I32_F32 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI4ToOffsetF32 v82, v5"), + "VOP3-encoded V_CVT_OFF_F32_I4 did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FractF32 v25"), + "VOP3-encoded VOP1 fract did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "RcpF32 v26"), + "VOP3-encoded VOP1 reciprocal did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 v29"), + "VOP3-encoded VOP1 not did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 v30"), + "VOP3-encoded VOP1 bit reverse did not lower through shared IR"); + Check( + Common::ContainsStr(result.ir_dump, "FindMsbFromHighU32 v33"), + "VOP3-encoded VOP1 find-first-bit-high did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FindLsbU32 v34"), + "VOP3-encoded VOP1 find-first-bit-low did not lower through shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareGtF32"), + "VOPC float compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtF32 exec_lo"), + "VOPC float compare-and-mask did not lower to exec mask IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtU32 exec_lo"), + "VOPC uint compare-and-mask did not lower to exec mask IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareFalse vcc_lo, v6, v6"), + "VOPC false compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareTrue vcc_lo, v6, v6"), + "VOPC true compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareOrderedF32 vcc_lo, v6, v6"), + "VOPC ordered compare did not lower to shared IR"); + Check( + Common::ContainsStr(result.ir_dump, "CompareUnorderedF32 vcc_lo, v6, v6"), + "VOPC unordered compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordLtF32 vcc_lo, v6, v6"), + "VOPC unordered-less compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordEqF32 vcc_lo, v6, v6"), + "VOPC unordered-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordLeF32 vcc_lo, v6, v6"), + "VOPC unordered-less-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordGtF32 vcc_lo, v6, v6"), + "VOPC unordered-greater compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordNeF32 vcc_lo, v6, v6"), + "VOPC unordered-not-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, + "CompareUnordNeF32 s0, 0x3f000000, v1"), + "VOPC SDWA unordered-not-equal compare did not lower to " + "scalar-destination IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareUnordGeF32 vcc_lo, v6, v6"), + "VOPC unordered-greater-equal compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, + "CompareMaskUnordLtF32 exec_lo, v6, v6"), + "VOPC unordered-less compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, + "CompareMaskUnordEqF32 exec_lo, v6, v6"), + "VOPC unordered-equal compare-and-mask did not lower to shared IR"); + Check( + Common::ContainsStr(result.ir_dump, + "CompareMaskUnordLeF32 exec_lo, v6, v6"), + "VOPC unordered-less-equal compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, + "CompareMaskUnordGtF32 exec_lo, v6, v6"), + "VOPC unordered-greater compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, + "CompareMaskUnordNeF32 exec_lo, v6, v6"), + "VOPC unordered-not-equal compare-and-mask did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, + "CompareMaskUnordGeF32 exec_lo, v6, v6"), + "VOPC unordered-greater-equal compare-and-mask did not lower to shared " + "IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareFalse vcc_lo, v5, v5"), + "VOPC integer false compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareTrue vcc_lo, v5, v5"), + "VOPC integer true compare did not lower to shared IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v8"), + "VOP3 mad did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v4, v1.neg, v9, s4"), + "VOP3 mad source modifiers did not lower to IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "CubeIdF32 v110, v5, v6, v7"), + "V_CUBEID_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeScF32 v111, v5, v6, v7"), + "V_CUBESC_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeTcF32 v112, v5, v6, v7"), + "V_CUBETC_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeMaF32 v113, v5, v6, v7"), + "V_CUBEMA_F32 did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "CubeMaF32 v5, v2, v0, v6.neg"), + "V_CUBEMA_F32 source modifier did not lower to cube IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v37"), + "VOP3 fma did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMin3F32 v38"), + "VOP3 min3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMax3F32 v39"), + "VOP3 max3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMed3F32 v40"), + "VOP3 med3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMin3I32 v41"), + "VOP3 signed min3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMin3U32 v42"), + "VOP3 unsigned min3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMax3I32 v43"), + "VOP3 signed max3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMax3U32 v44"), + "VOP3 unsigned max3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMed3I32 v45"), + "VOP3 signed med3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMed3U32 v46"), + "VOP3 unsigned med3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtract3U32 v47"), + "VOP3 unsigned bitfield extract did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtract3I32 v48"), + "VOP3 signed bitfield extract did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldInsertSelectU32 v49"), + "VOP3 bitfield insert-select did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AlignBitU32 v50"), + "VOP3 alignbit did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IAdd3U32 v51"), + "VOP3 add3 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMulU32 v52, v5, v6"), + "V_MUL_LO_U32 did not lower to multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "UMulHighU32 v53, v5, v6"), + "V_MUL_HI_U32 did not lower to high-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "SMulHighI32 v114, v5, v6"), + "V_MUL_HI_I32 did not lower to signed high-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndOrU32 v54, v5, v6, v7"), + "V_AND_OR_B32 did not lower to ternary bitwise IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOr3U32 v55, v5, v6, v7"), + "V_OR3_B32 did not lower to ternary bitwise IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXor3U32 v56, v5, v6, v7"), + "V_XOR3_B32 did not lower to ternary bitwise IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftAddU32 v57, v5, v6, v7"), + "V_LSHL_ADD_U32 did not lower to shared shift-left-add IR"); + Check(Common::ContainsStr(result.ir_dump, "AddShiftLeftU32 v58, v5, v6, v7"), + "V_ADD_LSHL_U32 did not lower to add-shift-left IR"); + Check(Common::ContainsStr(result.ir_dump, "XorAddU32 v59, v5, v6, v7"), + "V_XAD_U32 did not lower to xor-add IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftOrU32 v60, v5, v6, v7"), + "V_LSHL_OR_B32 did not lower to shift-left-or IR"); + Check(Common::ContainsStr(result.ir_dump, "SadU32 v61, v5, v6, v7"), + "V_SAD_U32 did not lower to sad IR"); + Check(Common::ContainsStr(result.ir_dump, "IMadI24U32 v62, v5, v6, v7"), + "V_MAD_I32_I24 did not lower to signed 24-bit mad IR"); + Check(Common::ContainsStr(result.ir_dump, "UMadU24U32 v63, v5, v6, v7"), + "V_MAD_U32_U24 did not lower to unsigned 24-bit mad IR"); + Check(Common::ContainsStr(result.ir_dump, "IMulU32 v69, v5, v6"), + "V_MUL_LO_I32 did not lower to multiply IR"); + Check(Common::ContainsStr(result.ir_dump, + "IAddCarryU32 v70, s0, v5, v6, 0x00000000"), + "V_ADD_I32 did not lower to carry-out add IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubBorrowU32 v71, s0, v5, v6"), + "V_SUB_I32 did not lower to borrow-out subtract IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubBorrowU32 v72, s0, v6, v5"), + "V_SUBREV_I32 did not reverse source order in borrow-out IR"); + Check(Common::ContainsStr(result.ir_dump, + "IMaxI16 v7.sdwa(sel=4,sext=0), 0x0000ffff, " + "v5.opsel(lo=1,hi=0,neghi=0)"), + "V_MAX_I16 did not lower to signed halfword max IR"); + Check(Common::ContainsStr(result.ir_dump, + "IMinI16 v7.sdwa(sel=5,sext=0), 0x00000002, " + "v4.opsel(lo=1,hi=0,neghi=0)"), + "V_MIN_I16 did not lower to signed halfword min IR"); + Check(Common::ContainsStr(result.ir_dump, + "BitFieldMaskU32 v73, 0x00000001, 0x00000004"), + "V_BFM_B32 did not lower to shared bitfield-mask IR"); + Check(Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v75, v6, v6"), + "V_CVT_PKRTZ_F16_F32 did not lower to shared pack IR"); + Check( + Common::ContainsStr(result.ir_dump, "PackF32ToF16Rtz v0, v0.neg, v1.neg"), + "V_CVT_PKRTZ_F16_F32 source modifiers did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v76, v6, 0x00000001"), + "V_LDEXP_F32 did not lower to ldexp IR"); + Check(Common::ContainsStr(result.ir_dump, "LdexpF32 v7, v9.abs, 0xfffffffe"), + "V_LDEXP_F32 source modifier did not lower to IR"); + Check( + Common::ContainsStr(result.ir_dump, "LdexpF32 v13.clamp, v6, 0xfffffffc"), + "V_LDEXP_F32 clamp modifier did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCountAddU32 v77, v5, v6"), + "VOP3 V_BCNT_U32_B32 did not lower to bit-count-add IR"); + Check(Common::ContainsStr(result.ir_dump, "PackSnorm2x16F32 v78, v6, v6"), + "V_CVT_PKNORM_I16_F32 did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "PackUnorm2x16F32 v79, v6, v6"), + "V_CVT_PKNORM_U16_F32 did not lower to shared pack IR"); + Check(Common::ContainsStr(result.ir_dump, "PackU16U32 v80, v5, v7"), + "V_CVT_PK_U16_U32 did not lower to shared pack IR"); + Check(SpirvContainsOpcode(result.spirv, 112), + "SPIR-V binary does not contain OpConvertUToF"); + Check(SpirvContainsOpcode(result.spirv, 109), + "SPIR-V binary does not contain OpConvertFToU"); + Check(SpirvContainsOpcode(result.spirv, 111), + "SPIR-V binary does not contain OpConvertSToF"); + Check(SpirvContainsOpcode(result.spirv, 110), + "SPIR-V binary does not contain OpConvertFToS"); + Check(SpirvContainsOpcode(result.spirv, 136), + "SPIR-V binary does not contain OpFDiv"); + Check(SpirvContainsOpcode(result.spirv, 127), + "SPIR-V binary does not contain OpFNegate"); + Check(SpirvContainsOpcode(result.spirv, 12), + "SPIR-V binary does not contain OpExtInst"); + Check(SpirvContainsExtInst(result.spirv, 58), + "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16"); + Check(SpirvContainsExtInst(result.spirv, 50), + "SPIR-V binary does not contain GLSL.std.450 Fma"); + Check(SpirvContainsExtInst(result.spirv, 43), + "SPIR-V binary does not contain GLSL.std.450 FClamp"); + Check(SpirvContainsExtInst(result.spirv, 53), + "SPIR-V binary does not contain GLSL.std.450 Ldexp"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd"); + Check(SpirvContainsOpcode(result.spirv, 149), + "SPIR-V binary does not contain OpIAddCarry"); + Check(SpirvContainsOpcode(result.spirv, 130), + "SPIR-V binary does not contain OpISub"); + Check(SpirvContainsOpcode(result.spirv, 132), + "SPIR-V binary does not contain OpIMul"); + Check(SpirvContainsOpcode(result.spirv, 171), + "SPIR-V binary does not contain OpINotEqual"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot"); + Check(SpirvContainsOpcode(result.spirv, 204), + "SPIR-V binary does not contain OpBitReverse"); + Check(SpirvContainsOpcode(result.spirv, 205), + "SPIR-V binary does not contain OpBitCount"); + Check(SpirvContainsOpcode(result.spirv, 166), + "SPIR-V binary does not contain OpLogicalOr"); + Check(SpirvContainsOpcode(result.spirv, 167), + "SPIR-V binary does not contain OpLogicalAnd"); + Check(SpirvContainsOpcode(result.spirv, 186), + "SPIR-V binary does not contain OpFOrdGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 181), + "SPIR-V binary does not contain OpFUnordEqual"); + Check(SpirvContainsOpcode(result.spirv, 183), + "SPIR-V binary does not contain OpFUnordNotEqual"); + Check(SpirvContainsOpcode(result.spirv, 185), + "SPIR-V binary does not contain OpFUnordLessThan"); + Check(SpirvContainsOpcode(result.spirv, 187), + "SPIR-V binary does not contain OpFUnordGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 189), + "SPIR-V binary does not contain OpFUnordLessThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 191), + "SPIR-V binary does not contain OpFUnordGreaterThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 190), + "SPIR-V binary does not contain OpFOrdGreaterThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 184), + "SPIR-V binary does not contain OpFOrdLessThan"); + Check(SpirvContainsOpcode(result.spirv, 173), + "SPIR-V binary does not contain OpSGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 174), + "SPIR-V binary does not contain OpUGreaterThanEqual"); + Check(SpirvContainsOpcode(result.spirv, 177), + "SPIR-V binary does not contain OpSLessThan"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain OpBitFieldSExtract"); + Check(SpirvContainsOpcode(result.spirv, 203), + "SPIR-V binary does not contain OpBitFieldUExtract"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for cmpx old exec mask"); + Check(SpirvContainsOpcode(result.spirv, 133), + "SPIR-V binary does not contain OpFMul"); + Check(SpirvContainsOpcode(result.spirv, 129), + "SPIR-V binary does not contain OpFAdd"); + Check(SpirvContainsOpcode(result.spirv, 151), + "SPIR-V binary does not contain OpUMulExtended"); + Check(SpirvContainsOpcode(result.spirv, 152), + "SPIR-V binary does not contain OpSMulExtended"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr"); + Check(SpirvContainsOpcode(result.spirv, 198), + "SPIR-V binary does not contain OpBitwiseXor"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain OpCompositeConstruct"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain OpCompositeExtract"); + Check(SpirvContainsOpcode(result.spirv, 339), + "SPIR-V binary does not contain OpGroupNonUniformBallot"); + Check(SpirvContainsOpcode(result.spirv, 343), + "SPIR-V binary does not contain OpGroupNonUniformBallotFindLSB"); + Check(SpirvContainsOpcode(result.spirv, 345), + "SPIR-V binary does not contain OpGroupNonUniformShuffle"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerExpandedAluBatch() { - const uint32_t shader[] = { - EncodeSopk(0x00, 9, 7), // s_movk_i32 s9, 7 - EncodeSopk(0x0f, 9, 2), // s_add_i32 s9, s9, 2 - EncodeSopk(0x10, 9, 3), // s_mulk_i32 s9, s9, 3 - EncodeSop2(0x07, 10, 9, 128), // s_min_u32 s10, s9, 0 - EncodeSop2(0x09, 11, 10, 130), // s_max_u32 s11, s10, 2 - EncodeSop2(0x26, 12, 11, 130), // s_mul_i32 s12, s11, 2 - EncodeVop2(0x05, 1, 242, 0), // v_subrev_f32 v1, 1.0, v0 - EncodeVop2(0x0f, 2, 1 + 256, 0), // v_min_f32 v2, v1, v0 - EncodeVop2(0x10, 3, 1 + 256, 2), // v_max_f32 v3, v1, v2 - EncodeVop2(0x27, 4, 130, 3), // v_subrev_nc_u32 v4, 2, v3 - EncodeVop2(0x16, 5, 129, 4), // v_lshrrev_b32 v5, 1, v4 - EncodeVop2(0x1a, 6, 129, 5), // v_lshlrev_b32 v6, 1, v5 - EncodeVop2(0x13, 7, 6 + 256, 5), // v_min_u32 v7, v6, v5 - EncodeVop2(0x14, 8, 7 + 256, 6), // v_max_u32 v8, v7, v6 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopk(0x00, 9, 7), // s_movk_i32 s9, 7 + EncodeSopk(0x0f, 9, 2), // s_add_i32 s9, s9, 2 + EncodeSopk(0x10, 9, 3), // s_mulk_i32 s9, s9, 3 + EncodeSop2(0x07, 10, 9, 128), // s_min_u32 s10, s9, 0 + EncodeSop2(0x09, 11, 10, 130), // s_max_u32 s11, s10, 2 + EncodeSop2(0x26, 12, 11, 130), // s_mul_i32 s12, s11, 2 + EncodeVop2(0x05, 1, 242, 0), // v_subrev_f32 v1, 1.0, v0 + EncodeVop2(0x0f, 2, 1 + 256, 0), // v_min_f32 v2, v1, v0 + EncodeVop2(0x10, 3, 1 + 256, 2), // v_max_f32 v3, v1, v2 + EncodeVop2(0x27, 4, 130, 3), // v_subrev_nc_u32 v4, 2, v3 + EncodeVop2(0x16, 5, 129, 4), // v_lshrrev_b32 v5, 1, v4 + EncodeVop2(0x1a, 6, 129, 5), // v_lshlrev_b32 v6, 1, v5 + EncodeVop2(0x13, 7, 6 + 256, 5), // v_min_u32 v7, v6, v5 + EncodeVop2(0x14, 8, 7 + 256, 6), // v_max_u32 v8, v7, v6 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_min_u32"), - "new decoder did not decode S_MIN_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_mulk_i32"), - "new decoder did not decode S_MULK_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_subrev_f32"), - "new decoder did not decode V_SUBREV_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b32"), - "new decoder did not decode V_LSHLREV_B32"); - Check(Common::ContainsStr(result.ir_dump, "IMulU32 s9, s9, 0x00000003"), - "SOPK multiply did not lower to self-multiply IR"); - Check(Common::ContainsStr(result.ir_dump, "UMinU32 s10"), "S_MIN_U32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "UMaxU32 s11"), "S_MAX_U32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FSubF32 v1, v0, 0x3f800000"), - "V_SUBREV_F32 did not reverse source order in IR"); - Check(Common::ContainsStr(result.ir_dump, "FMinF32 v2"), "V_MIN_F32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "FMaxF32 v3"), "V_MAX_F32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ISubU32 v4, v3, 0x00000002"), - "V_SUBREV_NC_U32 did not reverse source order in IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU32 v6, v5, 0x00000001"), - "V_LSHLREV_B32 did not reverse source order in IR"); - Check(Common::ContainsStr(result.ir_dump, "UMinU32 v7"), "V_MIN_U32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "UMaxU32 v8"), "V_MAX_U32 did not lower to IR"); - Check(SpirvContainsOpcode(result.spirv, 132), "SPIR-V binary does not contain OpIMul"); - Check(SpirvContainsOpcode(result.spirv, 169), "SPIR-V binary does not contain OpSelect"); - Check(SpirvContainsOpcode(result.spirv, 172), "SPIR-V binary does not contain OpUGreaterThan"); - Check(SpirvContainsOpcode(result.spirv, 176), "SPIR-V binary does not contain OpULessThan"); - Check(SpirvContainsOpcode(result.spirv, 184), "SPIR-V binary does not contain OpFOrdLessThan"); - Check(SpirvContainsOpcode(result.spirv, 186), - "SPIR-V binary does not contain OpFOrdGreaterThan"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_min_u32"), + "new decoder did not decode S_MIN_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_mulk_i32"), + "new decoder did not decode S_MULK_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_subrev_f32"), + "new decoder did not decode V_SUBREV_F32"); + Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b32"), + "new decoder did not decode V_LSHLREV_B32"); + Check(Common::ContainsStr(result.ir_dump, "IMulU32 s9, s9, 0x00000003"), + "SOPK multiply did not lower to self-multiply IR"); + Check(Common::ContainsStr(result.ir_dump, "UMinU32 s10"), + "S_MIN_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMaxU32 s11"), + "S_MAX_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FSubF32 v1, v0, 0x3f800000"), + "V_SUBREV_F32 did not reverse source order in IR"); + Check(Common::ContainsStr(result.ir_dump, "FMinF32 v2"), + "V_MIN_F32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "FMaxF32 v3"), + "V_MAX_F32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubU32 v4, v3, 0x00000002"), + "V_SUBREV_NC_U32 did not reverse source order in IR"); + Check(Common::ContainsStr(result.ir_dump, + "ShiftLeftLogicalU32 v6, v5, 0x00000001"), + "V_LSHLREV_B32 did not reverse source order in IR"); + Check(Common::ContainsStr(result.ir_dump, "UMinU32 v7"), + "V_MIN_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMaxU32 v8"), + "V_MAX_U32 did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 132), + "SPIR-V binary does not contain OpIMul"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect"); + Check(SpirvContainsOpcode(result.spirv, 172), + "SPIR-V binary does not contain OpUGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 176), + "SPIR-V binary does not contain OpULessThan"); + Check(SpirvContainsOpcode(result.spirv, 184), + "SPIR-V binary does not contain OpFOrdLessThan"); + Check(SpirvContainsOpcode(result.spirv, 186), + "SPIR-V binary does not contain OpFOrdGreaterThan"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerVop3pPackedF16() { - const uint32_t shader[] = { - EncodeVop3pWord0(0x0f, 114, 0, 7), - EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_add_f16 v114, v5, v6 - EncodeVop3pWord0(0x10, 115, 0, 7), - EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_mul_f16 v115, v5, v6 - EncodeVop3pWord0(0x11, 116, 0, 7), - EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_min_f16 v116, v5, v6 - EncodeVop3pWord0(0x12, 117, 0, 7), - EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_max_f16 v117, v5, v6 - EncodeVop3pWord0(0x0e, 118, 0, 7, 2), - EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7, 1), - EncodeVop3pWord0(0x20, 119), - EncodeVop3pWord1(6 + 256, 6 + 256, 6 + 256), // v_fma_f32 via VOP3P - 0xcc204044u, - 0x1a022110u, // v_fma_mix_f32 v68, v16.lo, v16.lo, 0.lo - EncodeVop3pWord0(0x21, 120, 0, 7), - EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7), // v_mad_mixlo_f16 - EncodeVop3pWord0(0x22, 121, 0, 7, 1), - EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7, 2), // v_mad_mixhi_f16 - EncodeVop3Word0(0x34b, 122, 0x0f), - EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), // native v_fma_f16 - EncodeVop3pWord0(0x22, 126, 0, 1, 0, true), - EncodeVop3pWord1(242, 243, 3 + 256, 1), // clamped v_mad_mixhi_f16 - EncodeVop3Word0(0x34b, 127, 0x8, 0, true), - EncodeVop3Word1(240, 4 + 256, 241), // clamped native high-half v_fma_f16 - 0xd711001au, - 0x0002170au, // v_pack_b32_f16 v26, v10, v11 - 0xd711182bu, - 0x0002170au, // v_pack_b32_f16 v43, v10.hi, v11.hi - 0x78765714u, // v_pk_fmac_f16 from boot shader 0x0000001c00f5c000 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeVop3pWord0(0x0f, 114, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_add_f16 v114, v5, v6 + EncodeVop3pWord0(0x10, 115, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_mul_f16 v115, v5, v6 + EncodeVop3pWord0(0x11, 116, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_min_f16 v116, v5, v6 + EncodeVop3pWord0(0x12, 117, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 0, 7), // v_pk_max_f16 v117, v5, v6 + EncodeVop3pWord0(0x0e, 118, 0, 7, 2), + EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7, 1), + EncodeVop3pWord0(0x20, 119), + EncodeVop3pWord1(6 + 256, 6 + 256, 6 + 256), // v_fma_f32 via VOP3P + 0xcc204044u, + 0x1a022110u, // v_fma_mix_f32 v68, v16.lo, v16.lo, 0.lo + EncodeVop3pWord0(0x21, 120, 0, 7), + EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7), // v_mad_mixlo_f16 + EncodeVop3pWord0(0x22, 121, 0, 7, 1), + EncodeVop3pWord1(5 + 256, 6 + 256, 7 + 256, 7, 2), // v_mad_mixhi_f16 + EncodeVop3Word0(0x34b, 122, 0x0f), + EncodeVop3Word1(5 + 256, 6 + 256, 7 + 256), // native v_fma_f16 + EncodeVop3pWord0(0x22, 126, 0, 1, 0, true), + EncodeVop3pWord1(242, 243, 3 + 256, 1), // clamped v_mad_mixhi_f16 + EncodeVop3Word0(0x34b, 127, 0x8, 0, true), + EncodeVop3Word1(240, 4 + 256, 241), // clamped native high-half v_fma_f16 + 0xd711001au, + 0x0002170au, // v_pack_b32_f16 v26, v10, v11 + 0xd711182bu, + 0x0002170au, // v_pack_b32_f16 v43, v10.hi, v11.hi + 0x78765714u, // v_pk_fmac_f16 from boot shader 0x0000001c00f5c000 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_f16 v114"), - "new decoder did not decode old-backed V_PK_ADD_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v115"), - "new decoder did not decode old-backed V_PK_MUL_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_min_f16 v116"), - "new decoder did not decode old-backed V_PK_MIN_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_max_f16 v117"), - "new decoder did not decode old-backed V_PK_MAX_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_fma_f16 v118"), - "new decoder did not decode old-backed V_PK_FMA_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v119"), - "new decoder did not decode old-backed VOP3P V_FMA_F32"); - Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v68, v16.opsel(lo=0,hi=1,neghi=0)"), - "new decoder did not decode VOP3P V_FMA_MIX_F32 source selectors"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixlo_f16 v120"), - "new decoder did not decode old-backed V_MAD_MIXLO_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixhi_f16 v121"), - "new decoder did not decode old-backed V_MAD_MIXHI_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_fma_f16 v122"), - "new decoder did not decode native VOP3 V_FMA_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixhi_f16 v126.sdwa(sel=5"), - "new decoder did not decode clamped V_MAD_MIXHI_F16 high-half destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_fma_f16 v127.sdwa(sel=5"), - "new decoder did not decode clamped native high-half V_FMA_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pack_b32_f16 v26"), - "new decoder did not decode native V_PACK_B32_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pack_b32_f16 v43") && - Common::ContainsStr(result.decoded_dump, "v10.opsel(lo=1"), - "new decoder did not decode V_PACK_B32_F16 source lane selectors"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_fmac_f16 v59"), - "new decoder did not decode VOP2 V_PK_FMAC_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v6.opsel(lo=0,hi=1,neghi=1)"), - "VOP3P high-lane source modifier was not dumped"); - Check(Common::ContainsStr(result.decoded_dump, "v120.sdwa(sel=4"), - "V_MAD_MIXLO_F16 did not expose low-half destination merge"); - Check(Common::ContainsStr(result.decoded_dump, "v121.sdwa(sel=5"), - "V_MAD_MIXHI_F16 did not expose high-half destination merge"); - Check(Common::ContainsStr(result.ir_dump, "PackedAddF16 v114"), - "V_PK_ADD_F16 did not lower to packed f16 IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedMulF16 v115"), - "V_PK_MUL_F16 did not lower to packed f16 IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedMinF16 v116"), - "V_PK_MIN_F16 did not lower to packed f16 IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedMaxF16 v117"), - "V_PK_MAX_F16 did not lower to packed f16 IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedFmaF16 v118"), - "V_PK_FMA_F16 did not lower to packed f16 IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v119"), - "VOP3P V_FMA_F32 did not lower through shared fma IR"); - Check(Common::ContainsStr(result.ir_dump, "FMadF32 v68, v16.opsel(lo=0,hi=1,neghi=0)"), - "VOP3P V_FMA_MIX_F32 source selectors did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v120.sdwa(sel=4"), - "V_MAD_MIXLO_F16 did not lower to shared mad-mix IR"); - Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v121.sdwa(sel=5"), - "V_MAD_MIXHI_F16 did not lower to shared mad-mix IR"); - Check(Common::ContainsStr(result.ir_dump, "FmaF16 v122"), - "native VOP3 V_FMA_F16 did not lower to native f16 fma IR"); - Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v126.sdwa(sel=5"), - "clamped V_MAD_MIXHI_F16 did not lower to shared mad-mix IR"); - Check(Common::ContainsStr(result.ir_dump, "FmaF16 v127.sdwa(sel=5"), - "clamped native V_FMA_F16 did not lower to native f16 fma IR"); - Check(Common::ContainsStr(result.ir_dump, "PackB32F16 v26"), - "V_PACK_B32_F16 did not lower to packed-bit IR"); - Check(Common::ContainsStr(result.ir_dump, "PackB32F16 v43") && - Common::ContainsStr(result.ir_dump, "v10.opsel(lo=1"), - "V_PACK_B32_F16 source selectors did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedFmaF16 v59, v20, v43, v59"), - "V_PK_FMAC_F16 did not lower using destination as packed FMA accumulator"); - Check(SpirvContainsExtInst(result.spirv, 62), - "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16 for VOP3P"); - Check(SpirvContainsExtInst(result.spirv, 58), - "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16 for VOP3P"); - Check(SpirvContainsExtInst(result.spirv, 50), - "SPIR-V binary does not contain GLSL.std.450 Fma for VOP3P"); - Check(SpirvContainsExtInst(result.spirv, 43), - "SPIR-V binary does not contain GLSL.std.450 FClamp for clamped f16 ops"); - Check(!SpirvContainsExtInst(result.spirv, 37), - "SPIR-V binary still contains GLSL.std.450 FMin for VOP3P packed min"); - Check(!SpirvContainsExtInst(result.spirv, 40), - "SPIR-V binary still contains GLSL.std.450 FMax for VOP3P packed max"); - Check(SpirvContainsOpcode(result.spirv, 169), - "SPIR-V binary does not contain OpSelect for VOP3P packed min/max"); - Check(SpirvContainsOpcode(result.spirv, 184), - "SPIR-V binary does not contain OpFOrdLessThan for VOP3P packed min"); - Check(SpirvContainsOpcode(result.spirv, 190), - "SPIR-V binary does not contain OpFOrdGreaterThanEqual for VOP3P packed max"); - Check(SpirvContainsOpcode(result.spirv, 197), - "SPIR-V binary does not contain OpBitwiseOr for VOP3P packed min/max"); - Check(SpirvContainsOpcode(result.spirv, 199), - "SPIR-V binary does not contain OpBitwiseAnd for VOP3P packed min/max"); - Check(SpirvContainsOpcode(result.spirv, 129), "SPIR-V binary does not contain OpFAdd"); - Check(SpirvContainsOpcode(result.spirv, 133), "SPIR-V binary does not contain OpFMul"); - Check(SpirvContainsOpcode(result.spirv, 81), - "SPIR-V binary does not contain OpCompositeExtract"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_f16 v114"), + "new decoder did not decode old-backed V_PK_ADD_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v115"), + "new decoder did not decode old-backed V_PK_MUL_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_min_f16 v116"), + "new decoder did not decode old-backed V_PK_MIN_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_max_f16 v117"), + "new decoder did not decode old-backed V_PK_MAX_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_fma_f16 v118"), + "new decoder did not decode old-backed V_PK_FMA_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f32 v119"), + "new decoder did not decode old-backed VOP3P V_FMA_F32"); + Check(Common::ContainsStr(result.decoded_dump, + "v_fma_f32 v68, v16.opsel(lo=0,hi=1,neghi=0)"), + "new decoder did not decode VOP3P V_FMA_MIX_F32 source selectors"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixlo_f16 v120"), + "new decoder did not decode old-backed V_MAD_MIXLO_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_mad_mixhi_f16 v121"), + "new decoder did not decode old-backed V_MAD_MIXHI_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f16 v122"), + "new decoder did not decode native VOP3 V_FMA_F16"); + Check(Common::ContainsStr(result.decoded_dump, + "v_mad_mixhi_f16 v126.sdwa(sel=5"), + "new decoder did not decode clamped V_MAD_MIXHI_F16 high-half " + "destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_fma_f16 v127.sdwa(sel=5"), + "new decoder did not decode clamped native high-half V_FMA_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pack_b32_f16 v26"), + "new decoder did not decode native V_PACK_B32_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pack_b32_f16 v43") && + Common::ContainsStr(result.decoded_dump, "v10.opsel(lo=1"), + "new decoder did not decode V_PACK_B32_F16 source lane selectors"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_fmac_f16 v59"), + "new decoder did not decode VOP2 V_PK_FMAC_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v6.opsel(lo=0,hi=1,neghi=1)"), + "VOP3P high-lane source modifier was not dumped"); + Check(Common::ContainsStr(result.decoded_dump, "v120.sdwa(sel=4"), + "V_MAD_MIXLO_F16 did not expose low-half destination merge"); + Check(Common::ContainsStr(result.decoded_dump, "v121.sdwa(sel=5"), + "V_MAD_MIXHI_F16 did not expose high-half destination merge"); + Check(Common::ContainsStr(result.ir_dump, "PackedAddF16 v114"), + "V_PK_ADD_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedMulF16 v115"), + "V_PK_MUL_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedMinF16 v116"), + "V_PK_MIN_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedMaxF16 v117"), + "V_PK_MAX_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedFmaF16 v118"), + "V_PK_FMA_F16 did not lower to packed f16 IR"); + Check(Common::ContainsStr(result.ir_dump, "FMadF32 v119"), + "VOP3P V_FMA_F32 did not lower through shared fma IR"); + Check(Common::ContainsStr(result.ir_dump, + "FMadF32 v68, v16.opsel(lo=0,hi=1,neghi=0)"), + "VOP3P V_FMA_MIX_F32 source selectors did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v120.sdwa(sel=4"), + "V_MAD_MIXLO_F16 did not lower to shared mad-mix IR"); + Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v121.sdwa(sel=5"), + "V_MAD_MIXHI_F16 did not lower to shared mad-mix IR"); + Check(Common::ContainsStr(result.ir_dump, "FmaF16 v122"), + "native VOP3 V_FMA_F16 did not lower to native f16 fma IR"); + Check(Common::ContainsStr(result.ir_dump, "MadMixF16 v126.sdwa(sel=5"), + "clamped V_MAD_MIXHI_F16 did not lower to shared mad-mix IR"); + Check(Common::ContainsStr(result.ir_dump, "FmaF16 v127.sdwa(sel=5"), + "clamped native V_FMA_F16 did not lower to native f16 fma IR"); + Check(Common::ContainsStr(result.ir_dump, "PackB32F16 v26"), + "V_PACK_B32_F16 did not lower to packed-bit IR"); + Check(Common::ContainsStr(result.ir_dump, "PackB32F16 v43") && + Common::ContainsStr(result.ir_dump, "v10.opsel(lo=1"), + "V_PACK_B32_F16 source selectors did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedFmaF16 v59, v20, v43, v59"), + "V_PK_FMAC_F16 did not lower using destination as packed FMA " + "accumulator"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16 for VOP3P"); + Check(SpirvContainsExtInst(result.spirv, 58), + "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16 for VOP3P"); + Check(SpirvContainsExtInst(result.spirv, 50), + "SPIR-V binary does not contain GLSL.std.450 Fma for VOP3P"); + Check( + SpirvContainsExtInst(result.spirv, 43), + "SPIR-V binary does not contain GLSL.std.450 FClamp for clamped f16 ops"); + Check(!SpirvContainsExtInst(result.spirv, 37), + "SPIR-V binary still contains GLSL.std.450 FMin for VOP3P packed min"); + Check(!SpirvContainsExtInst(result.spirv, 40), + "SPIR-V binary still contains GLSL.std.450 FMax for VOP3P packed max"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect for VOP3P packed min/max"); + Check(SpirvContainsOpcode(result.spirv, 184), + "SPIR-V binary does not contain OpFOrdLessThan for VOP3P packed min"); + Check(SpirvContainsOpcode(result.spirv, 190), + "SPIR-V binary does not contain OpFOrdGreaterThanEqual for VOP3P " + "packed max"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for VOP3P packed min/max"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd for VOP3P packed min/max"); + Check(SpirvContainsOpcode(result.spirv, 129), + "SPIR-V binary does not contain OpFAdd"); + Check(SpirvContainsOpcode(result.spirv, 133), + "SPIR-V binary does not contain OpFMul"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain OpCompositeExtract"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerStagedShaderOps() { - const uint32_t shader[] = { - EncodeSMovB32(0, 132), // s0 = 4 - EncodeSMovB32(1, 129), // s1 = 1 - EncodeSop2(0x05, 2, 0, 1), // s_subb_u32 s2, s0, s1 - EncodeSop1(0x1b, 3, 1), // s_bitset0_b32 s3, s1 - EncodeVop2(0x36, 70, 5 + 256, 6), // v_fmac_f16 v70, v5, v6 - EncodeVop2(0x37, 71, 7 + 256, 8), - 0x3c003c00u, // v_fmamk_f16 - EncodeVop2(0x38, 72, 9 + 256, 10), - 0x40004000u, // v_fmaak_f16 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 132), // s0 = 4 + EncodeSMovB32(1, 129), // s1 = 1 + EncodeSop2(0x05, 2, 0, 1), // s_subb_u32 s2, s0, s1 + EncodeSop1(0x1b, 3, 1), // s_bitset0_b32 s3, s1 + EncodeVop2(0x36, 70, 5 + 256, 6), // v_fmac_f16 v70, v5, v6 + EncodeVop2(0x37, 71, 7 + 256, 8), + 0x3c003c00u, // v_fmamk_f16 + EncodeVop2(0x38, 72, 9 + 256, 10), + 0x40004000u, // v_fmaak_f16 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_subb_u32 s2, s0, s1"), - "new decoder did not decode RDNA2 S_SUBB_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_bitset0_b32 s3, s1"), - "new decoder did not decode RDNA2 S_BITSET0_B32"); - Check(Common::ContainsStr(result.decoded_dump, "v_fmac_f16 v70.sdwa(sel=4"), - "new decoder did not decode RDNA2 V_FMAC_F16"); - Check(Common::ContainsStr(result.decoded_dump, - "v_fmamk_f16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"), - "new decoder did not consume V_FMAMK_F16 literal as the multiply source"); - Check(Common::ContainsStr(result.decoded_dump, - "v_fmaak_f16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"), - "new decoder did not consume V_FMAAK_F16 literal as the add source"); - Check(Common::ContainsStr(result.ir_dump, "ScalarSubBorrowCarryU32 s2, s0, s1, scc"), - "S_SUBB_U32 did not lower to scalar subtract-with-borrow IR"); - Check(Common::ContainsStr(result.ir_dump, "BitClearU32 s3, s3, s1"), - "S_BITSET0_B32 did not lower to bit-clear IR using the destination as input"); - Check(Common::ContainsStr(result.ir_dump, - "FmaF16 v70.sdwa(sel=4,sext=0), v5, v6, v70.sdwa(sel=4,sext=0)"), - "V_FMAC_F16 did not lower using the destination as the FMA accumulator"); - Check(Common::ContainsStr(result.ir_dump, "FmaF16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"), - "V_FMAMK_F16 did not lower with the literal in source 1"); - Check(Common::ContainsStr(result.ir_dump, "FmaF16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"), - "V_FMAAK_F16 did not lower with the literal in source 2"); - Check(SpirvContainsOpcode(result.spirv, 130), - "SPIR-V binary does not contain OpISub for S_SUBB_U32"); - Check(SpirvContainsOpcode(result.spirv, 166), - "SPIR-V binary does not contain OpLogicalOr for S_SUBB_U32 borrow-out"); - Check(SpirvContainsOpcode(result.spirv, 172), - "SPIR-V binary does not contain OpUGreaterThan for S_SUBB_U32 borrow-out"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical for S_BITSET0_B32"); - Check(SpirvContainsOpcode(result.spirv, 199), - "SPIR-V binary does not contain OpBitwiseAnd for S_BITSET0_B32"); - Check(SpirvContainsOpcode(result.spirv, 200), - "SPIR-V binary does not contain OpNot for S_BITSET0_B32"); - Check(SpirvContainsExtInst(result.spirv, 50), - "SPIR-V binary does not contain GLSL.std.450 Fma for VOP2 F16 FMA ops"); - Check(SpirvContainsExtInst(result.spirv, 58), - "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16 for VOP2 F16 FMA ops"); - Check(SpirvContainsExtInst(result.spirv, 62), - "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16 for VOP2 F16 FMA ops"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_subb_u32 s2, s0, s1"), + "new decoder did not decode RDNA2 S_SUBB_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitset0_b32 s3, s1"), + "new decoder did not decode RDNA2 S_BITSET0_B32"); + Check(Common::ContainsStr(result.decoded_dump, "v_fmac_f16 v70.sdwa(sel=4"), + "new decoder did not decode RDNA2 V_FMAC_F16"); + Check( + Common::ContainsStr( + result.decoded_dump, + "v_fmamk_f16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"), + "new decoder did not consume V_FMAMK_F16 literal as the multiply source"); + Check(Common::ContainsStr( + result.decoded_dump, + "v_fmaak_f16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"), + "new decoder did not consume V_FMAAK_F16 literal as the add source"); + Check(Common::ContainsStr(result.ir_dump, + "ScalarSubBorrowCarryU32 s2, s0, s1, scc"), + "S_SUBB_U32 did not lower to scalar subtract-with-borrow IR"); + Check(Common::ContainsStr(result.ir_dump, "BitClearU32 s3, s3, s1"), + "S_BITSET0_B32 did not lower to bit-clear IR using the destination as " + "input"); + Check( + Common::ContainsStr( + result.ir_dump, + "FmaF16 v70.sdwa(sel=4,sext=0), v5, v6, v70.sdwa(sel=4,sext=0)"), + "V_FMAC_F16 did not lower using the destination as the FMA accumulator"); + Check( + Common::ContainsStr(result.ir_dump, + "FmaF16 v71.sdwa(sel=4,sext=0), v7, 0x3c003c00, v8"), + "V_FMAMK_F16 did not lower with the literal in source 1"); + Check( + Common::ContainsStr(result.ir_dump, + "FmaF16 v72.sdwa(sel=4,sext=0), v9, v10, 0x40004000"), + "V_FMAAK_F16 did not lower with the literal in source 2"); + Check(SpirvContainsOpcode(result.spirv, 130), + "SPIR-V binary does not contain OpISub for S_SUBB_U32"); + Check(SpirvContainsOpcode(result.spirv, 166), + "SPIR-V binary does not contain OpLogicalOr for S_SUBB_U32 borrow-out"); + Check(SpirvContainsOpcode(result.spirv, 172), + "SPIR-V binary does not contain OpUGreaterThan for S_SUBB_U32 " + "borrow-out"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for S_BITSET0_B32"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd for S_BITSET0_B32"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot for S_BITSET0_B32"); + Check(SpirvContainsExtInst(result.spirv, 50), + "SPIR-V binary does not contain GLSL.std.450 Fma for VOP2 F16 FMA ops"); + Check(SpirvContainsExtInst(result.spirv, 58), + "SPIR-V binary does not contain GLSL.std.450 PackHalf2x16 for VOP2 F16 " + "FMA ops"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not contain GLSL.std.450 UnpackHalf2x16 for VOP2 " + "F16 FMA ops"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerBootF16UnaryOpcodes() { - const uint32_t shader[] = { - EncodeVop1(0x55, 4, 249), - EncodeVop1Sdwa(3, 5, 2, 6), - EncodeVop1(0x5b, 5, 4 + 256), - 0x7e12b8f9u, - 0x0006150bu, - 0x7e12b90cu, - EncodeVop1(0x5d, 6, 5 + 256), - 0x7e0e02f9u, - 0x008616c1u, - 0x7e1002f9u, - 0x008616c1u, - EncodeVop1(0x5e, 3, 3 + 256), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeVop1(0x55, 4, 249), + EncodeVop1Sdwa(3, 5, 2, 6), + EncodeVop1(0x5b, 5, 4 + 256), + 0x7e12b8f9u, + 0x0006150bu, + 0x7e12b90cu, + EncodeVop1(0x5d, 6, 5 + 256), + 0x7e0e02f9u, + 0x008616c1u, + 0x7e1002f9u, + 0x008616c1u, + EncodeVop1(0x5e, 3, 3 + 256), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "v_sqrt_f16 v4.sdwa(sel=5"), - "new decoder did not decode V_SQRT_F16 SDWA high-half destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_rndne_f16 v3"), - "new decoder did not decode V_RNDNE_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_floor_f16 v5"), - "new decoder did not decode V_FLOOR_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_ceil_f16 v9.sdwa(sel=5"), - "new decoder did not decode boot V_CEIL_F16 SDWA high-half destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_ceil_f16 v9"), - "new decoder did not decode boot V_CEIL_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_trunc_f16 v6"), - "new decoder did not decode V_TRUNC_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v7, -1") && - Common::ContainsStr(result.decoded_dump, "v_mov_b32 v8, -1"), - "new decoder did not accept full-width V_MOV_B32 SDWA with DST_U=PRESERVE"); - Check(!Common::ContainsStr(result.decoded_dump, - "VOP1 SDWA destination selector is not supported"), - "full-width V_MOV_B32 SDWA with DST_U=PRESERVE was rejected"); - Check(!Common::ContainsStr(result.decoded_dump, "unsupported family=VOP2 opcode=0x00"), - "VOP1 SDWA extension word was decoded as a phantom VOP2 instruction"); - Check(Common::ContainsStr(result.ir_dump, "SqrtF16 v4.sdwa(sel=5"), - "V_SQRT_F16 did not lower to f16 sqrt IR"); - Check(Common::ContainsStr(result.ir_dump, "FloorF16 v5"), - "V_FLOOR_F16 did not lower to f16 floor IR"); - Check(Common::ContainsStr(result.ir_dump, "CeilF16 v9.sdwa(sel=5"), - "boot V_CEIL_F16 SDWA did not lower to f16 ceil IR"); - Check(Common::ContainsStr(result.ir_dump, "CeilF16 v9"), - "V_CEIL_F16 did not lower to f16 ceil IR"); - Check(Common::ContainsStr(result.ir_dump, "TruncF16 v6"), - "V_TRUNC_F16 did not lower to f16 trunc IR"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 v7, 0xffffffff") && - Common::ContainsStr(result.ir_dump, "MoveU32 v8, 0xffffffff"), - "full-width V_MOV_B32 SDWA with DST_U=PRESERVE did not lower to move IR"); - Check(Common::ContainsStr(result.ir_dump, "RoundEvenF16 v3"), - "V_RNDNE_F16 did not lower to f16 round-even IR"); - Check(SpirvContainsExtInst(result.spirv, 31), - "SPIR-V binary does not contain GLSL.std.450 Sqrt for V_SQRT_F16"); - Check(SpirvContainsExtInst(result.spirv, 8), - "SPIR-V binary does not contain GLSL.std.450 Floor for V_FLOOR_F16"); - Check(SpirvContainsExtInst(result.spirv, 9), - "SPIR-V binary does not contain GLSL.std.450 Ceil for V_CEIL_F16"); - Check(SpirvContainsExtInst(result.spirv, 3), - "SPIR-V binary does not contain GLSL.std.450 Trunc for V_TRUNC_F16"); - Check(SpirvContainsExtInst(result.spirv, 2), - "SPIR-V binary does not contain GLSL.std.450 RoundEven for V_RNDNE_F16"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_sqrt_f16 v4.sdwa(sel=5"), + "new decoder did not decode V_SQRT_F16 SDWA high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_rndne_f16 v3"), + "new decoder did not decode V_RNDNE_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_floor_f16 v5"), + "new decoder did not decode V_FLOOR_F16"); + Check( + Common::ContainsStr(result.decoded_dump, "v_ceil_f16 v9.sdwa(sel=5"), + "new decoder did not decode boot V_CEIL_F16 SDWA high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_ceil_f16 v9"), + "new decoder did not decode boot V_CEIL_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_trunc_f16 v6"), + "new decoder did not decode V_TRUNC_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_mov_b32 v7, -1") && + Common::ContainsStr(result.decoded_dump, "v_mov_b32 v8, -1"), + "new decoder did not accept full-width V_MOV_B32 SDWA with " + "DST_U=PRESERVE"); + Check(!Common::ContainsStr(result.decoded_dump, + "VOP1 SDWA destination selector is not supported"), + "full-width V_MOV_B32 SDWA with DST_U=PRESERVE was rejected"); + Check(!Common::ContainsStr(result.decoded_dump, + "unsupported family=VOP2 opcode=0x00"), + "VOP1 SDWA extension word was decoded as a phantom VOP2 instruction"); + Check(Common::ContainsStr(result.ir_dump, "SqrtF16 v4.sdwa(sel=5"), + "V_SQRT_F16 did not lower to f16 sqrt IR"); + Check(Common::ContainsStr(result.ir_dump, "FloorF16 v5"), + "V_FLOOR_F16 did not lower to f16 floor IR"); + Check(Common::ContainsStr(result.ir_dump, "CeilF16 v9.sdwa(sel=5"), + "boot V_CEIL_F16 SDWA did not lower to f16 ceil IR"); + Check(Common::ContainsStr(result.ir_dump, "CeilF16 v9"), + "V_CEIL_F16 did not lower to f16 ceil IR"); + Check(Common::ContainsStr(result.ir_dump, "TruncF16 v6"), + "V_TRUNC_F16 did not lower to f16 trunc IR"); + Check( + Common::ContainsStr(result.ir_dump, "MoveU32 v7, 0xffffffff") && + Common::ContainsStr(result.ir_dump, "MoveU32 v8, 0xffffffff"), + "full-width V_MOV_B32 SDWA with DST_U=PRESERVE did not lower to move IR"); + Check(Common::ContainsStr(result.ir_dump, "RoundEvenF16 v3"), + "V_RNDNE_F16 did not lower to f16 round-even IR"); + Check(SpirvContainsExtInst(result.spirv, 31), + "SPIR-V binary does not contain GLSL.std.450 Sqrt for V_SQRT_F16"); + Check(SpirvContainsExtInst(result.spirv, 8), + "SPIR-V binary does not contain GLSL.std.450 Floor for V_FLOOR_F16"); + Check(SpirvContainsExtInst(result.spirv, 9), + "SPIR-V binary does not contain GLSL.std.450 Ceil for V_CEIL_F16"); + Check(SpirvContainsExtInst(result.spirv, 3), + "SPIR-V binary does not contain GLSL.std.450 Trunc for V_TRUNC_F16"); + Check( + SpirvContainsExtInst(result.spirv, 2), + "SPIR-V binary does not contain GLSL.std.450 RoundEven for V_RNDNE_F16"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerBootB16PackedAndSdwaOpcodes() { - const uint32_t shader[] = { - 0xd7070001u, - 0x00020081u, // v_lshrrev_b16 v1.lo, 1, v0.lo - 0xd7071008u, - 0x0002128fu, // v_lshrrev_b16 with selected high source lane - 0xd7144005u, - 0x00020481u, // v_lshlrev_b16 v5.hi, 1, v2.lo - 0xd7034001u, - 0x0002066au, // v_add_nc_u16 v1.hi, v106.lo, v3.lo - 0xd70e4011u, - 0x00020e80u, // v_sub_nc_i16 v17.hi, 0, v7.lo - 0xcc03400eu, - 0x10020e80u, // v_pk_sub_i16 from boot shader 0x0000001980eb0000 - 0xcc0a4104u, - 0x300202ffu, - 0x00007fffu, // v_pk_add_u16 with literal - 0x360e04f9u, - 0x04861482u, // v_and_b32 SDWA partial low-half destination - 0x381212f9u, - 0x04041508u, // v_or_b32 SDWA partial high-half destination - 0xcc10c00cu, - 0x18021d0du, // clamped v_pk_mul_f16 - 0x7e08a0f9u, - 0x00061500u, // v_cvt_f16_u16 v4.hi, v0 from boot shader - EncodeVop1(0x50, 16, 10 + 256), // v_cvt_f16_u16 v16, v10 - 0x7e1ca307u, // v_cvt_f16_i16 v14, v7 - 0x7e1ca2f9u, - 0x00051511u, // v_cvt_f16_i16 v14.hi, v17.hi - 0x7e08a704u, // v_cvt_i16_f16 v4, v4 from boot shader - EncodeVop1(0x52, 15, 4 + 256), // v_cvt_u16_f16 v15, v4 - 0x4a0a08f9u, - 0x0c860688u, // v_add_nc_u32 v5, 8, sign-extended v4.lo - 0x4a0c0cf9u, - 0x0d860688u, // v_add_nc_u32 v6, 8, sign-extended v6.hi - 0x4c1616f9u, - 0x0686128du, // v_sub_nc_u32 v11.byte2, 13, v11; preserve other destination bytes - 0x261418f9u, - 0x0686149fu, // v_min_u32 v10.word0, 31, v12; preserve upper destination word - 0xbf810000u, - }; + const uint32_t shader[] = { + 0xd7070001u, + 0x00020081u, // v_lshrrev_b16 v1.lo, 1, v0.lo + 0xd7071008u, + 0x0002128fu, // v_lshrrev_b16 with selected high source lane + 0xd7144005u, + 0x00020481u, // v_lshlrev_b16 v5.hi, 1, v2.lo + 0xd7034001u, + 0x0002066au, // v_add_nc_u16 v1.hi, v106.lo, v3.lo + 0xd70e4011u, + 0x00020e80u, // v_sub_nc_i16 v17.hi, 0, v7.lo + 0xcc03400eu, + 0x10020e80u, // v_pk_sub_i16 from boot shader 0x0000001980eb0000 + 0xcc0a4104u, + 0x300202ffu, + 0x00007fffu, // v_pk_add_u16 with literal + 0x360e04f9u, + 0x04861482u, // v_and_b32 SDWA partial low-half destination + 0x381212f9u, + 0x04041508u, // v_or_b32 SDWA partial high-half destination + 0xcc10c00cu, + 0x18021d0du, // clamped v_pk_mul_f16 + 0x7e08a0f9u, + 0x00061500u, // v_cvt_f16_u16 v4.hi, v0 from boot shader + EncodeVop1(0x50, 16, 10 + 256), // v_cvt_f16_u16 v16, v10 + 0x7e1ca307u, // v_cvt_f16_i16 v14, v7 + 0x7e1ca2f9u, + 0x00051511u, // v_cvt_f16_i16 v14.hi, v17.hi + 0x7e08a704u, // v_cvt_i16_f16 v4, v4 from boot shader + EncodeVop1(0x52, 15, 4 + 256), // v_cvt_u16_f16 v15, v4 + 0x4a0a08f9u, + 0x0c860688u, // v_add_nc_u32 v5, 8, sign-extended v4.lo + 0x4a0c0cf9u, + 0x0d860688u, // v_add_nc_u32 v6, 8, sign-extended v6.hi + 0x4c1616f9u, + 0x0686128du, // v_sub_nc_u32 v11.byte2, 13, v11; preserve other + // destination bytes + 0x261418f9u, + 0x0686149fu, // v_min_u32 v10.word0, 31, v12; preserve upper destination + // word + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "v_lshrrev_b16 v1.sdwa(sel=4"), - "new decoder did not decode low-half V_LSHRREV_B16"); - Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b16 v5.sdwa(sel=5"), - "new decoder did not decode high-half V_LSHLREV_B16"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u16 v1.sdwa(sel=5"), - "new decoder did not decode V_ADD_NC_U16 op_sel destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_sub_nc_i16 v17.sdwa(sel=5"), - "new decoder did not decode V_SUB_NC_I16 op_sel destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_sub_i16 v14"), - "new decoder did not decode V_PK_SUB_I16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_u16 v4"), - "new decoder did not decode V_PK_ADD_U16"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_u16 v4, 0x00007fff"), - "V_PK_ADD_U16 literal was not consumed as a source operand"); - Check(Common::ContainsStr(result.decoded_dump, "v_and_b32 v7.sdwa(sel=4"), - "new decoder did not decode partial-destination V_AND_B32 SDWA"); - Check(Common::ContainsStr(result.decoded_dump, "v_or_b32 v9.sdwa(sel=5"), - "new decoder did not decode partial-destination V_OR_B32 SDWA"); - Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v12"), - "new decoder did not decode clamped V_PK_MUL_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_u16 v4.sdwa(sel=5"), - "new decoder did not decode/consume SDWA V_CVT_F16_U16"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_u16 v16"), - "new decoder did not decode plain V_CVT_F16_U16"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_i16 v14"), - "new decoder did not decode plain V_CVT_F16_I16"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_i16 v14.sdwa(sel=5"), - "new decoder did not decode V_CVT_F16_I16 SDWA high-half destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_i16_f16 v4"), - "new decoder did not decode plain V_CVT_I16_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_cvt_u16_f16 v15"), - "new decoder still rejects plain V_CVT_U16_F16"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v5") && - Common::ContainsStr(result.decoded_dump, "v4.sdwa(sel=4,sext=1"), - "new decoder did not decode V_ADD_NC_U32 SDWA sign-extended low word"); - Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v6") && - Common::ContainsStr(result.decoded_dump, "v6.sdwa(sel=5,sext=1"), - "new decoder did not decode V_ADD_NC_U32 SDWA sign-extended high word"); - Check(Common::ContainsStr(result.decoded_dump, "v_sub_nc_u32 v11.sdwa(sel=2"), - "new decoder did not decode V_SUB_NC_U32 SDWA byte-2 destination"); - Check(Common::ContainsStr(result.decoded_dump, "v_min_u32 v10.sdwa(sel=4"), - "new decoder did not decode V_MIN_U32 SDWA low-word destination"); - Check(!Common::ContainsStr(result.decoded_dump, "unsupported family=VOP2 opcode=0x00"), - "literal/SDWA extension words were decoded as phantom VOP2 instructions"); - Check(!Common::ContainsStr(result.decoded_dump, - "VOP2 SDWA destination selector is not supported"), - "partial bitwise SDWA destination is still rejected"); - Check(Common::ContainsStr(result.ir_dump, "ShiftRightLogicalU16"), - "V_LSHRREV_B16 did not lower to 16-bit logical right shift IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU16"), - "V_LSHLREV_B16 did not lower to 16-bit logical left shift IR"); - Check(Common::ContainsStr(result.ir_dump, "IAddU16 v1.sdwa(sel=5"), - "V_ADD_NC_U16 did not lower to 16-bit add IR"); - Check(Common::ContainsStr(result.ir_dump, "ISubI16 v17.sdwa(sel=5"), - "V_SUB_NC_I16 did not lower to 16-bit subtract IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedSubI16 v14"), - "V_PK_SUB_I16 did not lower to packed I16 subtract IR"); - Check(Common::ContainsStr(result.ir_dump, "PackedAddU16 v4"), - "V_PK_ADD_U16 did not lower to packed U16 add IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertI16ToF16 v14"), - "V_CVT_F16_I16 did not lower to signed I16-to-F16 conversion IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertU16ToF16 v4.sdwa(sel=5"), - "V_CVT_F16_U16 SDWA did not lower to unsigned U16-to-F16 conversion IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertU16ToF16 v16"), - "V_CVT_F16_U16 plain form did not lower to unsigned U16-to-F16 conversion IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToI16 v4"), - "V_CVT_I16_F16 did not lower to signed F16-to-I16 conversion IR"); - Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToU16 v15"), - "V_CVT_U16_F16 did not lower to unsigned F16-to-U16 conversion IR"); - Check(Common::ContainsStr(result.ir_dump, "IAddU32 v5") && - Common::ContainsStr(result.ir_dump, "v4.sdwa(sel=4,sext=1"), - "V_ADD_NC_U32 SDWA sign-extended low word did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IAddU32 v6") && - Common::ContainsStr(result.ir_dump, "v6.sdwa(sel=5,sext=1"), - "V_ADD_NC_U32 SDWA sign-extended high word did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ISubU32 v11.sdwa(sel=2"), - "V_SUB_NC_U32 SDWA byte-2 destination did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "UMinU32 v10.sdwa(sel=4"), - "V_MIN_U32 SDWA low-word destination did not lower to IR"); - Check(SpirvContainsOpcode(result.spirv, 128), - "SPIR-V binary does not contain OpIAdd for U16 operations"); - Check(SpirvContainsOpcode(result.spirv, 202), - "SPIR-V binary does not contain OpBitFieldSExtract for SDWA sign extension"); - Check(SpirvContainsOpcode(result.spirv, 130), - "SPIR-V binary does not contain OpISub for I16 operations"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical for B16 shift"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical for B16 shift"); - Check(SpirvContainsOpcode(result.spirv, 197), - "SPIR-V binary does not contain OpBitwiseOr for packed U16 result"); - Check(SpirvContainsOpcode(result.spirv, 124), - "SPIR-V binary does not contain OpConvertSToF for V_CVT_F16_I16"); - Check(SpirvContainsOpcode(result.spirv, 109), - "SPIR-V binary does not contain OpConvertFToU for V_CVT_U16_F16"); - Check(SpirvContainsOpcode(result.spirv, 110), - "SPIR-V binary does not contain OpConvertFToS for V_CVT_I16_F16"); - Check(SpirvContainsOpcode(result.spirv, 112), - "SPIR-V binary does not contain OpConvertUToF for V_CVT_F16_U16"); - Check(SpirvContainsExtInst(result.spirv, 43), - "SPIR-V binary does not contain GLSL.std.450 FClamp for clamped V_PK_MUL_F16"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_lshrrev_b16 v1.sdwa(sel=4"), + "new decoder did not decode low-half V_LSHRREV_B16"); + Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b16 v5.sdwa(sel=5"), + "new decoder did not decode high-half V_LSHLREV_B16"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u16 v1.sdwa(sel=5"), + "new decoder did not decode V_ADD_NC_U16 op_sel destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_nc_i16 v17.sdwa(sel=5"), + "new decoder did not decode V_SUB_NC_I16 op_sel destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_sub_i16 v14"), + "new decoder did not decode V_PK_SUB_I16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_u16 v4"), + "new decoder did not decode V_PK_ADD_U16"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_u16 v4, 0x00007fff"), + "V_PK_ADD_U16 literal was not consumed as a source operand"); + Check(Common::ContainsStr(result.decoded_dump, "v_and_b32 v7.sdwa(sel=4"), + "new decoder did not decode partial-destination V_AND_B32 SDWA"); + Check(Common::ContainsStr(result.decoded_dump, "v_or_b32 v9.sdwa(sel=5"), + "new decoder did not decode partial-destination V_OR_B32 SDWA"); + Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v12"), + "new decoder did not decode clamped V_PK_MUL_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_u16 v4.sdwa(sel=5"), + "new decoder did not decode/consume SDWA V_CVT_F16_U16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_u16 v16"), + "new decoder did not decode plain V_CVT_F16_U16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_f16_i16 v14"), + "new decoder did not decode plain V_CVT_F16_I16"); + Check( + Common::ContainsStr(result.decoded_dump, "v_cvt_f16_i16 v14.sdwa(sel=5"), + "new decoder did not decode V_CVT_F16_I16 SDWA high-half destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_i16_f16 v4"), + "new decoder did not decode plain V_CVT_I16_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_cvt_u16_f16 v15"), + "new decoder still rejects plain V_CVT_U16_F16"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v5") && + Common::ContainsStr(result.decoded_dump, "v4.sdwa(sel=4,sext=1"), + "new decoder did not decode V_ADD_NC_U32 SDWA sign-extended low word"); + Check(Common::ContainsStr(result.decoded_dump, "v_add_nc_u32 v6") && + Common::ContainsStr(result.decoded_dump, "v6.sdwa(sel=5,sext=1"), + "new decoder did not decode V_ADD_NC_U32 SDWA sign-extended high word"); + Check(Common::ContainsStr(result.decoded_dump, "v_sub_nc_u32 v11.sdwa(sel=2"), + "new decoder did not decode V_SUB_NC_U32 SDWA byte-2 destination"); + Check(Common::ContainsStr(result.decoded_dump, "v_min_u32 v10.sdwa(sel=4"), + "new decoder did not decode V_MIN_U32 SDWA low-word destination"); + Check( + !Common::ContainsStr(result.decoded_dump, + "unsupported family=VOP2 opcode=0x00"), + "literal/SDWA extension words were decoded as phantom VOP2 instructions"); + Check(!Common::ContainsStr(result.decoded_dump, + "VOP2 SDWA destination selector is not supported"), + "partial bitwise SDWA destination is still rejected"); + Check(Common::ContainsStr(result.ir_dump, "ShiftRightLogicalU16"), + "V_LSHRREV_B16 did not lower to 16-bit logical right shift IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU16"), + "V_LSHLREV_B16 did not lower to 16-bit logical left shift IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU16 v1.sdwa(sel=5"), + "V_ADD_NC_U16 did not lower to 16-bit add IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubI16 v17.sdwa(sel=5"), + "V_SUB_NC_I16 did not lower to 16-bit subtract IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedSubI16 v14"), + "V_PK_SUB_I16 did not lower to packed I16 subtract IR"); + Check(Common::ContainsStr(result.ir_dump, "PackedAddU16 v4"), + "V_PK_ADD_U16 did not lower to packed U16 add IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertI16ToF16 v14"), + "V_CVT_F16_I16 did not lower to signed I16-to-F16 conversion IR"); + Check( + Common::ContainsStr(result.ir_dump, "ConvertU16ToF16 v4.sdwa(sel=5"), + "V_CVT_F16_U16 SDWA did not lower to unsigned U16-to-F16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertU16ToF16 v16"), + "V_CVT_F16_U16 plain form did not lower to unsigned U16-to-F16 " + "conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToI16 v4"), + "V_CVT_I16_F16 did not lower to signed F16-to-I16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "ConvertF16ToU16 v15"), + "V_CVT_U16_F16 did not lower to unsigned F16-to-U16 conversion IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU32 v5") && + Common::ContainsStr(result.ir_dump, "v4.sdwa(sel=4,sext=1"), + "V_ADD_NC_U32 SDWA sign-extended low word did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IAddU32 v6") && + Common::ContainsStr(result.ir_dump, "v6.sdwa(sel=5,sext=1"), + "V_ADD_NC_U32 SDWA sign-extended high word did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ISubU32 v11.sdwa(sel=2"), + "V_SUB_NC_U32 SDWA byte-2 destination did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "UMinU32 v10.sdwa(sel=4"), + "V_MIN_U32 SDWA low-word destination did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd for U16 operations"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain OpBitFieldSExtract for SDWA sign " + "extension"); + Check(SpirvContainsOpcode(result.spirv, 130), + "SPIR-V binary does not contain OpISub for I16 operations"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical for B16 shift"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for B16 shift"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for packed U16 result"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain OpConvertSToF for V_CVT_F16_I16"); + Check(SpirvContainsOpcode(result.spirv, 109), + "SPIR-V binary does not contain OpConvertFToU for V_CVT_U16_F16"); + Check(SpirvContainsOpcode(result.spirv, 110), + "SPIR-V binary does not contain OpConvertFToS for V_CVT_I16_F16"); + Check(SpirvContainsOpcode(result.spirv, 112), + "SPIR-V binary does not contain OpConvertUToF for V_CVT_F16_U16"); + Check(SpirvContainsExtInst(result.spirv, 43), + "SPIR-V binary does not contain GLSL.std.450 FClamp for clamped " + "V_PK_MUL_F16"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerScalarB64Alu() { - const uint32_t shader[] = { - EncodeSMovB32(0, 129), // s0 = 1 - EncodeSMovB32(1, 130), // s1 = 2 - EncodeSop1(0x07, 41, 0), // s_not_b32 s41, s0 - EncodeSop1(0x04, 2, 0), // s_mov_b64 s[2:3], s[0:1] - EncodeSop1(0x08, 4, 2), // s_not_b64 s[4:5], s[2:3] - EncodeSop2(0x0f, 6, 2, 4), // s_and_b64 s[6:7], s[2:3], s[4:5] - EncodeSop2(0x11, 8, 6, 2), // s_or_b64 s[8:9], s[6:7], s[2:3] - EncodeSop2(0x13, 10, 8, 4), // s_xor_b64 s[10:11], s[8:9], s[4:5] - EncodeSop2(0x14, 36, 0, 1), // s_andn2_b32 s36, s0, s1 - EncodeSop2(0x16, 37, 0, 1), // s_orn2_b32 s37, s0, s1 - EncodeSop2(0x18, 38, 0, 1), // s_nand_b32 s38, s0, s1 - EncodeSop2(0x1a, 39, 0, 1), // s_nor_b32 s39, s0, s1 - EncodeSop2(0x1c, 40, 0, 1), // s_xnor_b32 s40, s0, s1 - EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 - EncodeSop2(0x0b, 12, 10, 2), // s_cselect_b64 s[12:13], s[10:11], s[2:3] - EncodeSop2(0x15, 14, 10, 4), // s_andn2_b64 s[14:15], s[10:11], s[4:5] - EncodeSop2(0x17, 16, 14, 6), // s_orn2_b64 s[16:17], s[14:15], s[6:7] - EncodeSop2(0x19, 18, 16, 8), // s_nand_b64 s[18:19], s[16:17], s[8:9] - EncodeSop2(0x1b, 20, 18, 10), // s_nor_b64 s[20:21], s[18:19], s[10:11] - EncodeSop2(0x1d, 22, 20, 12), // s_xnor_b64 s[22:23], s[20:21], s[12:13] - EncodeSop2(0x1f, 24, 22, 1), // s_lshl_b64 s[24:25], s[22:23], s1 - EncodeSop2(0x21, 26, 24, 0), // s_lshr_b64 s[26:27], s[24:25], s0 - EncodeSop2(0x25, 28, 132, 130), // s_bfm_b64 s[28:29], 4, 2 - EncodeSop1(0x10, 30, 26), // s_bcnt1_i32_b64 s30, s[26:27] - EncodeSop1(0x16, 106, 26), // s_flbit_i32_b64 vcc_lo, s[26:27] - EncodeSop1(0x3b, 32, 30), // s_bitreplicate_b64_b32 s[32:33], s30 - EncodeSop2(0x29, 34, 32, 255), // s_bfe_u64 s[34:35], s[32:33], 0x00040002 - 0x00040002u, - EncodeSop1(0x0a, 126, 126), // s_wqm_b64 exec, exec - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 129), // s0 = 1 + EncodeSMovB32(1, 130), // s1 = 2 + EncodeSop1(0x07, 41, 0), // s_not_b32 s41, s0 + EncodeSop1(0x04, 2, 0), // s_mov_b64 s[2:3], s[0:1] + EncodeSop1(0x08, 4, 2), // s_not_b64 s[4:5], s[2:3] + EncodeSop2(0x0f, 6, 2, 4), // s_and_b64 s[6:7], s[2:3], s[4:5] + EncodeSop2(0x11, 8, 6, 2), // s_or_b64 s[8:9], s[6:7], s[2:3] + EncodeSop2(0x13, 10, 8, 4), // s_xor_b64 s[10:11], s[8:9], s[4:5] + EncodeSop2(0x14, 36, 0, 1), // s_andn2_b32 s36, s0, s1 + EncodeSop2(0x16, 37, 0, 1), // s_orn2_b32 s37, s0, s1 + EncodeSop2(0x18, 38, 0, 1), // s_nand_b32 s38, s0, s1 + EncodeSop2(0x1a, 39, 0, 1), // s_nor_b32 s39, s0, s1 + EncodeSop2(0x1c, 40, 0, 1), // s_xnor_b32 s40, s0, s1 + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSop2(0x0b, 12, 10, 2), // s_cselect_b64 s[12:13], s[10:11], s[2:3] + EncodeSop2(0x15, 14, 10, 4), // s_andn2_b64 s[14:15], s[10:11], s[4:5] + EncodeSop2(0x17, 16, 14, 6), // s_orn2_b64 s[16:17], s[14:15], s[6:7] + EncodeSop2(0x19, 18, 16, 8), // s_nand_b64 s[18:19], s[16:17], s[8:9] + EncodeSop2(0x1b, 20, 18, 10), // s_nor_b64 s[20:21], s[18:19], s[10:11] + EncodeSop2(0x1d, 22, 20, 12), // s_xnor_b64 s[22:23], s[20:21], s[12:13] + EncodeSop2(0x1f, 24, 22, 1), // s_lshl_b64 s[24:25], s[22:23], s1 + EncodeSop2(0x21, 26, 24, 0), // s_lshr_b64 s[26:27], s[24:25], s0 + EncodeSop2(0x25, 28, 132, 130), // s_bfm_b64 s[28:29], 4, 2 + EncodeSop1(0x10, 30, 26), // s_bcnt1_i32_b64 s30, s[26:27] + EncodeSop1(0x16, 106, 26), // s_flbit_i32_b64 vcc_lo, s[26:27] + EncodeSop1(0x3b, 32, 30), // s_bitreplicate_b64_b32 s[32:33], s30 + EncodeSop2(0x29, 34, 32, 255), // s_bfe_u64 s[34:35], s[32:33], 0x00040002 + 0x00040002u, + EncodeSop1(0x0a, 126, 126), // s_wqm_b64 exec, exec + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 s2, s0"), - "new decoder did not decode old-backed S_MOV_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_not_b32 s41, s0"), - "new decoder did not decode RDNA2 S_NOT_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_not_b64 s4, s2"), - "new decoder did not decode old-backed S_NOT_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_and_b64 s6, s2, s4"), - "new decoder did not decode old-backed S_AND_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_or_b64 s8, s6, s2"), - "new decoder did not decode old-backed S_OR_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_xor_b64 s10, s8, s4"), - "new decoder did not decode old-backed S_XOR_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_andn2_b32 s36, s0, s1"), - "new decoder did not decode RDNA2 S_ANDN2_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_orn2_b32 s37, s0, s1"), - "new decoder did not decode RDNA2 S_ORN2_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_nand_b32 s38, s0, s1"), - "new decoder did not decode RDNA2 S_NAND_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_nor_b32 s39, s0, s1"), - "new decoder did not decode RDNA2 S_NOR_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_xnor_b32 s40, s0, s1"), - "new decoder did not decode RDNA2 S_XNOR_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b64 s12, s10, s2"), - "new decoder did not decode old-backed S_CSELECT_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_andn2_b64 s14, s10, s4"), - "new decoder did not decode old-backed S_ANDN2_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_orn2_b64 s16, s14, s6"), - "new decoder did not decode old-backed S_ORN2_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_nand_b64 s18, s16, s8"), - "new decoder did not decode old-backed S_NAND_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_nor_b64 s20, s18, s10"), - "new decoder did not decode old-backed S_NOR_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_xnor_b64 s22, s20, s12"), - "new decoder did not decode old-backed S_XNOR_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_lshl_b64 s24, s22, s1"), - "new decoder did not decode old-backed S_LSHL_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_lshr_b64 s26, s24, s0"), - "new decoder did not decode old-backed S_LSHR_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_bfm_b64 s28, 4, 2"), - "new decoder did not decode old-backed S_BFM_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_bcnt1_i32_b64 s30, s26"), - "new decoder did not decode old-backed S_BCNT1_I32_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_flbit_i32_b64 vcc_lo, s26"), - "new decoder did not decode RDNA2 S_FLBIT_I32_B64"); - Check(Common::ContainsStr(result.decoded_dump, "s_bitreplicate_b64_b32 s32, s30"), - "new decoder did not decode old-backed S_BITREPLICATE_B64_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_bfe_u64 s34, s32, 0x00040002"), - "new decoder did not decode old-backed S_BFE_U64"); - Check(Common::ContainsStr(result.decoded_dump, "s_wqm_b64 exec_lo, exec_lo"), - "new decoder did not decode old-backed S_WQM_B64"); - Check(Common::ContainsStr(result.ir_dump, "MoveU64 s2, s0"), - "S_MOV_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 s41, s0"), - "S_NOT_B32 did not lower to scalar bitwise-not IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU64 s4, s2"), - "S_NOT_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU64 s6, s2, s4"), - "S_AND_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseOrU64 s8, s6, s2"), - "S_OR_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseXorU64 s10, s8, s4"), - "S_XOR_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndNotU32 s36, s0, s1"), - "S_ANDN2_B32 did not lower to scalar bitwise-and-not IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseOrNotU32 s37, s0, s1"), - "S_ORN2_B32 did not lower to scalar bitwise-or-not IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNandU32 s38, s0, s1"), - "S_NAND_B32 did not lower to scalar bitwise-nand IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNorU32 s39, s0, s1"), - "S_NOR_B32 did not lower to scalar bitwise-nor IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 s40, s0, s1"), - "S_XNOR_B32 did not lower to scalar bitwise-xnor IR"); - Check(Common::ContainsStr(result.ir_dump, "SelectU64 s12"), - "S_CSELECT_B64 did not lower to paired-dword select IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndNotU64 s14, s10, s4"), - "S_ANDN2_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseOrNotU64 s16, s14, s6"), - "S_ORN2_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNandU64 s18, s16, s8"), - "S_NAND_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseNorU64 s20, s18, s10"), - "S_NOR_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU64 s22, s20, s12"), - "S_XNOR_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU64 s24, s22, s1"), - "S_LSHL_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftRightLogicalU64 s26, s24, s0"), - "S_LSHR_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU64 s28"), - "S_BFM_B64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitCountU64 s30, s26"), - "S_BCNT1_I32_B64 did not lower to paired-source bit count IR"); - Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU64 vcc_lo, s26"), - "S_FLBIT_I32_B64 did not lower to paired-source leading-zero IR"); - Check(Common::ContainsStr(result.ir_dump, "BitReplicateB64B32 s32, s30"), - "S_BITREPLICATE_B64_B32 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldExtractU64 s34, s32, 0x00040002"), - "S_BFE_U64 did not lower to paired-dword IR"); - Check(Common::ContainsStr(result.ir_dump, "WqmB64 exec_lo, exec_lo"), - "S_WQM_B64 did not lower to whole-quad-mask IR"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad for scalar B64 ops"); - Check(SpirvContainsOpcode(result.spirv, 62), - "SPIR-V binary does not contain OpStore for scalar B64 ops"); - Check(SpirvContainsOpcode(result.spirv, 197), - "SPIR-V binary does not contain OpBitwiseOr for scalar B64 ops"); - Check(SpirvContainsOpcode(result.spirv, 198), - "SPIR-V binary does not contain OpBitwiseXor for scalar B64 ops"); - Check(SpirvContainsOpcode(result.spirv, 199), - "SPIR-V binary does not contain OpBitwiseAnd for scalar B64 ops"); - Check(SpirvContainsOpcode(result.spirv, 200), - "SPIR-V binary does not contain OpNot for scalar B64 ops"); - Check(SpirvContainsOpcode(result.spirv, 169), - "SPIR-V binary does not contain OpSelect for scalar B64 select"); - Check(SpirvContainsOpcode(result.spirv, 171), - "SPIR-V binary does not contain OpINotEqual for scalar B64 whole-quad mask"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical for scalar B64 shifts"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical for scalar B64 shifts"); - Check(SpirvContainsOpcode(result.spirv, 201), - "SPIR-V binary does not contain OpBitFieldInsert for scalar B64 mask"); - Check(SpirvContainsOpcode(result.spirv, 203), - "SPIR-V binary does not contain OpBitFieldUExtract for scalar B64 extract"); - Check(SpirvContainsOpcode(result.spirv, 205), - "SPIR-V binary does not contain OpBitCount for scalar B64 bit count"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 s2, s0"), + "new decoder did not decode old-backed S_MOV_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_not_b32 s41, s0"), + "new decoder did not decode RDNA2 S_NOT_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_not_b64 s4, s2"), + "new decoder did not decode old-backed S_NOT_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_and_b64 s6, s2, s4"), + "new decoder did not decode old-backed S_AND_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_or_b64 s8, s6, s2"), + "new decoder did not decode old-backed S_OR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_xor_b64 s10, s8, s4"), + "new decoder did not decode old-backed S_XOR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_andn2_b32 s36, s0, s1"), + "new decoder did not decode RDNA2 S_ANDN2_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_orn2_b32 s37, s0, s1"), + "new decoder did not decode RDNA2 S_ORN2_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_nand_b32 s38, s0, s1"), + "new decoder did not decode RDNA2 S_NAND_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_nor_b32 s39, s0, s1"), + "new decoder did not decode RDNA2 S_NOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_xnor_b32 s40, s0, s1"), + "new decoder did not decode RDNA2 S_XNOR_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b64 s12, s10, s2"), + "new decoder did not decode old-backed S_CSELECT_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_andn2_b64 s14, s10, s4"), + "new decoder did not decode old-backed S_ANDN2_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_orn2_b64 s16, s14, s6"), + "new decoder did not decode old-backed S_ORN2_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_nand_b64 s18, s16, s8"), + "new decoder did not decode old-backed S_NAND_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_nor_b64 s20, s18, s10"), + "new decoder did not decode old-backed S_NOR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_xnor_b64 s22, s20, s12"), + "new decoder did not decode old-backed S_XNOR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshl_b64 s24, s22, s1"), + "new decoder did not decode old-backed S_LSHL_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_lshr_b64 s26, s24, s0"), + "new decoder did not decode old-backed S_LSHR_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfm_b64 s28, 4, 2"), + "new decoder did not decode old-backed S_BFM_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_bcnt1_i32_b64 s30, s26"), + "new decoder did not decode old-backed S_BCNT1_I32_B64"); + Check(Common::ContainsStr(result.decoded_dump, "s_flbit_i32_b64 vcc_lo, s26"), + "new decoder did not decode RDNA2 S_FLBIT_I32_B64"); + Check(Common::ContainsStr(result.decoded_dump, + "s_bitreplicate_b64_b32 s32, s30"), + "new decoder did not decode old-backed S_BITREPLICATE_B64_B32"); + Check(Common::ContainsStr(result.decoded_dump, + "s_bfe_u64 s34, s32, 0x00040002"), + "new decoder did not decode old-backed S_BFE_U64"); + Check(Common::ContainsStr(result.decoded_dump, "s_wqm_b64 exec_lo, exec_lo"), + "new decoder did not decode old-backed S_WQM_B64"); + Check(Common::ContainsStr(result.ir_dump, "MoveU64 s2, s0"), + "S_MOV_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU32 s41, s0"), + "S_NOT_B32 did not lower to scalar bitwise-not IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNotU64 s4, s2"), + "S_NOT_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU64 s6, s2, s4"), + "S_AND_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOrU64 s8, s6, s2"), + "S_OR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXorU64 s10, s8, s4"), + "S_XOR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndNotU32 s36, s0, s1"), + "S_ANDN2_B32 did not lower to scalar bitwise-and-not IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOrNotU32 s37, s0, s1"), + "S_ORN2_B32 did not lower to scalar bitwise-or-not IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNandU32 s38, s0, s1"), + "S_NAND_B32 did not lower to scalar bitwise-nand IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNorU32 s39, s0, s1"), + "S_NOR_B32 did not lower to scalar bitwise-nor IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU32 s40, s0, s1"), + "S_XNOR_B32 did not lower to scalar bitwise-xnor IR"); + Check(Common::ContainsStr(result.ir_dump, "SelectU64 s12"), + "S_CSELECT_B64 did not lower to paired-dword select IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseAndNotU64 s14, s10, s4"), + "S_ANDN2_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseOrNotU64 s16, s14, s6"), + "S_ORN2_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNandU64 s18, s16, s8"), + "S_NAND_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseNorU64 s20, s18, s10"), + "S_NOR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitwiseXnorU64 s22, s20, s12"), + "S_XNOR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftLeftLogicalU64 s24, s22, s1"), + "S_LSHL_B64 did not lower to paired-dword IR"); + Check( + Common::ContainsStr(result.ir_dump, "ShiftRightLogicalU64 s26, s24, s0"), + "S_LSHR_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU64 s28"), + "S_BFM_B64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCountU64 s30, s26"), + "S_BCNT1_I32_B64 did not lower to paired-source bit count IR"); + Check(Common::ContainsStr(result.ir_dump, "FindMsbFromHighU64 vcc_lo, s26"), + "S_FLBIT_I32_B64 did not lower to paired-source leading-zero IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReplicateB64B32 s32, s30"), + "S_BITREPLICATE_B64_B32 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, + "BitFieldExtractU64 s34, s32, 0x00040002"), + "S_BFE_U64 did not lower to paired-dword IR"); + Check(Common::ContainsStr(result.ir_dump, "WqmB64 exec_lo, exec_lo"), + "S_WQM_B64 did not lower to whole-quad-mask IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 198), + "SPIR-V binary does not contain OpBitwiseXor for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot for scalar B64 ops"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect for scalar B64 select"); + Check(SpirvContainsOpcode(result.spirv, 171), + "SPIR-V binary does not contain OpINotEqual for scalar B64 whole-quad " + "mask"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for scalar B64 " + "shifts"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical for scalar B64 " + "shifts"); + Check(SpirvContainsOpcode(result.spirv, 201), + "SPIR-V binary does not contain OpBitFieldInsert for scalar B64 mask"); + Check(SpirvContainsOpcode(result.spirv, 203), + "SPIR-V binary does not contain OpBitFieldUExtract for scalar B64 " + "extract"); + Check(SpirvContainsOpcode(result.spirv, 205), + "SPIR-V binary does not contain OpBitCount for scalar B64 bit count"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerSignedCompareAlu() { - const uint32_t shader[] = { - EncodeSMovB32(0, 193), // s0 = -1 - EncodeSMovB32(1, 129), // s1 = 1 - EncodeSopc(0x02, 0, 1), // s_cmp_gt_i32 s0, s1 - EncodeSopk(0x07, 0, -2), // s_cmp_lt_i32 s0, -2 - EncodeVopc(0x84, 0, 1), // v_cmp_gt_i32 s0, v1 - EncodeVopc(0x81, 0, 1), // v_cmp_lt_i32 s0, v1 - 0x7d130ef9u, - 0x86068413u, // v_cmp_lt_i16 vcc_lo.sdwa, v19.word0, v7.word1 - 0x7d1d02f9u, - 0x86068213u, // v_cmp_ge_i16 vcc_lo.sdwa, v19.word0, v1.word1 - EncodeVopc(0xa9, 0, 1), // v_cmp_lt_u16 s0, v1 - EncodeVopc(0x94, 0, 1), // v_cmpx_gt_i32 s0, v1 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 193), // s0 = -1 + EncodeSMovB32(1, 129), // s1 = 1 + EncodeSopc(0x02, 0, 1), // s_cmp_gt_i32 s0, s1 + EncodeSopk(0x07, 0, -2), // s_cmp_lt_i32 s0, -2 + EncodeVopc(0x84, 0, 1), // v_cmp_gt_i32 s0, v1 + EncodeVopc(0x81, 0, 1), // v_cmp_lt_i32 s0, v1 + 0x7d130ef9u, + 0x86068413u, // v_cmp_lt_i16 vcc_lo.sdwa, v19.word0, v7.word1 + 0x7d1d02f9u, + 0x86068213u, // v_cmp_ge_i16 vcc_lo.sdwa, v19.word0, v1.word1 + EncodeVopc(0xa9, 0, 1), // v_cmp_lt_u16 s0, v1 + EncodeVopc(0x94, 0, 1), // v_cmpx_gt_i32 s0, v1 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_i32"), - "new decoder did not decode SOPC signed compare"); - Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lt_i32"), - "new decoder did not decode SOPK signed compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_gt_i32"), - "new decoder did not decode VOPC signed greater-than compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_i32"), - "new decoder did not decode VOPC signed less-than compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_i16"), - "new decoder did not decode VOPC signed halfword less-than compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ge_i16"), - "new decoder did not decode VOPC signed halfword greater-or-equal compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_u16"), - "new decoder did not decode VOPC unsigned halfword less-than compare"); - Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_i32"), - "new decoder did not decode VOPC signed compare-and-mask"); - Check(Common::ContainsStr(result.ir_dump, "CompareGtI32"), - "signed greater-than compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareLtI32"), - "signed less-than compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareLtI16"), - "signed halfword less-than compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareGeI16"), - "signed halfword greater-or-equal compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareLtU16"), - "unsigned halfword less-than compare did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtI32 exec_lo"), - "signed compare-and-mask did not lower to exec mask IR"); - Check(SpirvContainsOpcode(result.spirv, 173), "SPIR-V binary does not contain OpSGreaterThan"); - Check(SpirvContainsOpcode(result.spirv, 177), "SPIR-V binary does not contain OpSLessThan"); - Check(SpirvContainsOpcode(result.spirv, 202), - "SPIR-V binary does not contain OpBitFieldSExtract for I16 compare"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_i32"), + "new decoder did not decode SOPC signed compare"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lt_i32"), + "new decoder did not decode SOPK signed compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_gt_i32"), + "new decoder did not decode VOPC signed greater-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_i32"), + "new decoder did not decode VOPC signed less-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_i16"), + "new decoder did not decode VOPC signed halfword less-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_ge_i16"), + "new decoder did not decode VOPC signed halfword greater-or-equal " + "compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_u16"), + "new decoder did not decode VOPC unsigned halfword less-than compare"); + Check(Common::ContainsStr(result.decoded_dump, "v_cmpx_gt_i32"), + "new decoder did not decode VOPC signed compare-and-mask"); + Check(Common::ContainsStr(result.ir_dump, "CompareGtI32"), + "signed greater-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtI32"), + "signed less-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtI16"), + "signed halfword less-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareGeI16"), + "signed halfword greater-or-equal compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtU16"), + "unsigned halfword less-than compare did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareMaskGtI32 exec_lo"), + "signed compare-and-mask did not lower to exec mask IR"); + Check(SpirvContainsOpcode(result.spirv, 173), + "SPIR-V binary does not contain OpSGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 177), + "SPIR-V binary does not contain OpSLessThan"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain OpBitFieldSExtract for I16 compare"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerSignedMinShiftAlu() { - const uint32_t shader[] = { - EncodeSMovB32(0, 193), // s0 = -1 - EncodeSMovB32(1, 130), // s1 = 2 - EncodeSop2(0x06, 2, 0, 1), // s_min_i32 s2, s0, s1 - EncodeSop2(0x08, 3, 0, 1), // s_max_i32 s3, s0, s1 - EncodeSop2(0x22, 4, 0, 129), // s_ashr_i32 s4, s0, 1 - EncodeVop2(0x11, 1, 0, 0), // v_min_i32 v1, s0, v0 - EncodeVop2(0x12, 2, 1, 1), // v_max_i32 v2, s1, v1 - EncodeVop2(0x17, 3, 1 + 256, 1), // v_ashr_i32 v3, v1, v1 - EncodeVop2(0x18, 4, 129, 3), // v_ashrrev_i32 v4, 1, v3 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 193), // s0 = -1 + EncodeSMovB32(1, 130), // s1 = 2 + EncodeSop2(0x06, 2, 0, 1), // s_min_i32 s2, s0, s1 + EncodeSop2(0x08, 3, 0, 1), // s_max_i32 s3, s0, s1 + EncodeSop2(0x22, 4, 0, 129), // s_ashr_i32 s4, s0, 1 + EncodeVop2(0x11, 1, 0, 0), // v_min_i32 v1, s0, v0 + EncodeVop2(0x12, 2, 1, 1), // v_max_i32 v2, s1, v1 + EncodeVop2(0x17, 3, 1 + 256, 1), // v_ashr_i32 v3, v1, v1 + EncodeVop2(0x18, 4, 129, 3), // v_ashrrev_i32 v4, 1, v3 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_min_i32"), - "new decoder did not decode S_MIN_I32"); - Check(Common::ContainsStr(result.decoded_dump, "s_ashr_i32"), - "new decoder did not decode S_ASHR_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_min_i32"), - "new decoder did not decode V_MIN_I32"); - Check(Common::ContainsStr(result.decoded_dump, "v_ashrrev_i32"), - "new decoder did not decode V_ASHRREV_I32"); - Check(Common::ContainsStr(result.ir_dump, "IMinI32 s2"), "S_MIN_I32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMaxI32 s3"), "S_MAX_I32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftRightArithmeticI32 s4"), - "S_ASHR_I32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMinI32 v1"), "V_MIN_I32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "IMaxI32 v2"), "V_MAX_I32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ShiftRightArithmeticI32 v4, v3, 0x00000001"), - "V_ASHRREV_I32 did not reverse source order in IR"); - Check(SpirvContainsOpcode(result.spirv, 173), "SPIR-V binary does not contain OpSGreaterThan"); - Check(SpirvContainsOpcode(result.spirv, 177), "SPIR-V binary does not contain OpSLessThan"); - Check(SpirvContainsOpcode(result.spirv, 195), - "SPIR-V binary does not contain OpShiftRightArithmetic"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_min_i32"), + "new decoder did not decode S_MIN_I32"); + Check(Common::ContainsStr(result.decoded_dump, "s_ashr_i32"), + "new decoder did not decode S_ASHR_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_min_i32"), + "new decoder did not decode V_MIN_I32"); + Check(Common::ContainsStr(result.decoded_dump, "v_ashrrev_i32"), + "new decoder did not decode V_ASHRREV_I32"); + Check(Common::ContainsStr(result.ir_dump, "IMinI32 s2"), + "S_MIN_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMaxI32 s3"), + "S_MAX_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ShiftRightArithmeticI32 s4"), + "S_ASHR_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMinI32 v1"), + "V_MIN_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "IMaxI32 v2"), + "V_MAX_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, + "ShiftRightArithmeticI32 v4, v3, 0x00000001"), + "V_ASHRREV_I32 did not reverse source order in IR"); + Check(SpirvContainsOpcode(result.spirv, 173), + "SPIR-V binary does not contain OpSGreaterThan"); + Check(SpirvContainsOpcode(result.spirv, 177), + "SPIR-V binary does not contain OpSLessThan"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerScalarBitfieldAlu() { - const uint32_t shader[] = { - EncodeSMovB32(0, 193), // s0 = -1 - EncodeSMovB32(1, 129), // s1 = 1 - EncodeSopc(0x02, 1, 0), // s_cmp_gt_i32 s1, s0 - EncodeSop2(0x0a, 2, 1, 0), // s_cselect_b32 s2, s1, s0 - EncodeSop1(0x34, 3, 0), // s_abs_i32 s3, s0 - EncodeSop1(0x0b, 4, 1), // s_brev_b32 s4, s1 - EncodeSop2(0x24, 5, 132, 129), // s_bfm_b32 s5, 4, 1 - EncodeSop2(0x27, 6, 4, 255), // s_bfe_u32 s6, s4, literal - 0x00080004u, - EncodeSop2(0x32, 7, 0, 1), // s_pack_ll_b32_b16 s7, s0, s1 - EncodeSop2(0x33, 8, 0, 1), // s_pack_lh_b32_b16 s8, s0, s1 - EncodeSop2(0x34, 9, 0, 1), // s_pack_hh_b32_b16 s9, s0, s1 - EncodeSopc(0x0c, 0, 1), // s_bitcmp0_b32 s0, s1 - EncodeSopc(0x0d, 1, 0), // s_bitcmp1_b32 s1, s0 - EncodeSopc(0x13, 0, 2), // s_cmp_lg_u64 s[0:1], s[2:3] - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 193), // s0 = -1 + EncodeSMovB32(1, 129), // s1 = 1 + EncodeSopc(0x02, 1, 0), // s_cmp_gt_i32 s1, s0 + EncodeSop2(0x0a, 2, 1, 0), // s_cselect_b32 s2, s1, s0 + EncodeSop1(0x34, 3, 0), // s_abs_i32 s3, s0 + EncodeSop1(0x0b, 4, 1), // s_brev_b32 s4, s1 + EncodeSop2(0x24, 5, 132, 129), // s_bfm_b32 s5, 4, 1 + EncodeSop2(0x27, 6, 4, 255), // s_bfe_u32 s6, s4, literal + 0x00080004u, + EncodeSop2(0x32, 7, 0, 1), // s_pack_ll_b32_b16 s7, s0, s1 + EncodeSop2(0x33, 8, 0, 1), // s_pack_lh_b32_b16 s8, s0, s1 + EncodeSop2(0x34, 9, 0, 1), // s_pack_hh_b32_b16 s9, s0, s1 + EncodeSopc(0x0c, 0, 1), // s_bitcmp0_b32 s0, s1 + EncodeSopc(0x0d, 1, 0), // s_bitcmp1_b32 s1, s0 + EncodeSopc(0x13, 0, 2), // s_cmp_lg_u64 s[0:1], s[2:3] + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b32"), - "new decoder did not decode S_CSELECT_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_abs_i32"), - "new decoder did not decode S_ABS_I32"); - Check(Common::ContainsStr(result.decoded_dump, "s_brev_b32"), - "new decoder did not decode S_BREV_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_bfm_b32"), - "new decoder did not decode S_BFM_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_bfe_u32"), - "new decoder did not decode S_BFE_U32"); - Check(Common::ContainsStr(result.decoded_dump, "s_pack_hh_b32_b16"), - "new decoder did not decode S_PACK_HH_B32_B16"); - Check(Common::ContainsStr(result.decoded_dump, "s_bitcmp0_b32"), - "new decoder did not decode S_BITCMP0_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_bitcmp1_b32"), - "new decoder did not decode S_BITCMP1_B32"); - Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lg_u64"), - "new decoder did not decode S_CMP_LG_U64"); - Check(Common::ContainsStr(result.ir_dump, "SelectU32 s2"), "S_CSELECT_B32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AbsI32 s3"), "S_ABS_I32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 s4"), - "S_BREV_B32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU32 s5"), - "S_BFM_B32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitFieldExtractU32 s6"), - "S_BFE_U32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "PackLowLowU16 s7"), - "S_PACK_LL_B32_B16 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "PackLowHighU16 s8"), - "S_PACK_LH_B32_B16 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "PackHighHighU16 s9"), - "S_PACK_HH_B32_B16 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitCompare0B32"), - "S_BITCMP0_B32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BitCompare1B32"), - "S_BITCMP1_B32 did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "CompareNeU64"), "S_CMP_LG_U64 did not lower to IR"); - Check(SpirvContainsOpcode(result.spirv, 126), "SPIR-V binary does not contain OpSNegate"); - Check(SpirvContainsOpcode(result.spirv, 169), "SPIR-V binary does not contain OpSelect"); - Check(SpirvContainsOpcode(result.spirv, 201), - "SPIR-V binary does not contain OpBitFieldInsert"); - Check(SpirvContainsOpcode(result.spirv, 203), - "SPIR-V binary does not contain OpBitFieldUExtract"); - Check(SpirvContainsOpcode(result.spirv, 171), "SPIR-V binary does not contain OpINotEqual"); - Check(SpirvContainsOpcode(result.spirv, 166), "SPIR-V binary does not contain OpLogicalOr"); - Check(SpirvContainsOpcode(result.spirv, 204), "SPIR-V binary does not contain OpBitReverse"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b32"), + "new decoder did not decode S_CSELECT_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_abs_i32"), + "new decoder did not decode S_ABS_I32"); + Check(Common::ContainsStr(result.decoded_dump, "s_brev_b32"), + "new decoder did not decode S_BREV_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfm_b32"), + "new decoder did not decode S_BFM_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bfe_u32"), + "new decoder did not decode S_BFE_U32"); + Check(Common::ContainsStr(result.decoded_dump, "s_pack_hh_b32_b16"), + "new decoder did not decode S_PACK_HH_B32_B16"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitcmp0_b32"), + "new decoder did not decode S_BITCMP0_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_bitcmp1_b32"), + "new decoder did not decode S_BITCMP1_B32"); + Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lg_u64"), + "new decoder did not decode S_CMP_LG_U64"); + Check(Common::ContainsStr(result.ir_dump, "SelectU32 s2"), + "S_CSELECT_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AbsI32 s3"), + "S_ABS_I32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitReverseU32 s4"), + "S_BREV_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldMaskU32 s5"), + "S_BFM_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitFieldExtractU32 s6"), + "S_BFE_U32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackLowLowU16 s7"), + "S_PACK_LL_B32_B16 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackLowHighU16 s8"), + "S_PACK_LH_B32_B16 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "PackHighHighU16 s9"), + "S_PACK_HH_B32_B16 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCompare0B32"), + "S_BITCMP0_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BitCompare1B32"), + "S_BITCMP1_B32 did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "CompareNeU64"), + "S_CMP_LG_U64 did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 126), + "SPIR-V binary does not contain OpSNegate"); + Check(SpirvContainsOpcode(result.spirv, 169), + "SPIR-V binary does not contain OpSelect"); + Check(SpirvContainsOpcode(result.spirv, 201), + "SPIR-V binary does not contain OpBitFieldInsert"); + Check(SpirvContainsOpcode(result.spirv, 203), + "SPIR-V binary does not contain OpBitFieldUExtract"); + Check(SpirvContainsOpcode(result.spirv, 171), + "SPIR-V binary does not contain OpINotEqual"); + Check(SpirvContainsOpcode(result.spirv, 166), + "SPIR-V binary does not contain OpLogicalOr"); + Check(SpirvContainsOpcode(result.spirv, 204), + "SPIR-V binary does not contain OpBitReverse"); + CheckSpirvBinaryValidates(result.spirv); } -void CheckNewDecoderUnsupported(const uint32_t* shader, uint32_t words, const char* family, - const char* opcode_name) { - ShaderRecompiler::Decoder::Program program; - std::string error; - const std::span code {shader, words}; - Check(ShaderRecompiler::Decoder::DecodeProgram(code, program, &error), error.c_str()); - Check(program.instructions.size() >= 2, "decoder did not return instruction plus endpgm"); - const auto text = ShaderRecompiler::Decoder::ProgramToString(program); - Check(Common::ContainsStr(text, family), - "decoder unsupported text did not include opcode family"); - Check(Common::ContainsStr(text, opcode_name), - "decoder unsupported text did not include opcode name"); +void CheckNewDecoderUnsupported(const uint32_t *shader, uint32_t words, + const char *family, const char *opcode_name) { + ShaderRecompiler::Decoder::Program program; + std::string error; + const std::span code{shader, words}; + Check(ShaderRecompiler::Decoder::DecodeProgram(code, program, &error), + error.c_str()); + Check(program.instructions.size() >= 2, + "decoder did not return instruction plus endpgm"); + const auto text = ShaderRecompiler::Decoder::ProgramToString(program); + Check(Common::ContainsStr(text, family), + "decoder unsupported text did not include opcode family"); + Check(Common::ContainsStr(text, opcode_name), + "decoder unsupported text did not include opcode name"); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - ShaderRecompiler::CompileResult result; - Check(!ShaderRecompiler::TryRecompile(code, options, result, &error), - "unsupported opcode unexpectedly lowered without implementation"); - Check(Common::ContainsStr(error, "no IR lowering") || - Common::ContainsStr(error, "unsupported decoded"), - "unsupported lowering error was not explicit"); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(!ShaderRecompiler::TryRecompile(code, options, result, &error), + "unsupported opcode unexpectedly lowered without implementation"); + Check(Common::ContainsStr(error, "no IR lowering") || + Common::ContainsStr(error, "unsupported decoded"), + "unsupported lowering error was not explicit"); } void TestNewShaderRecompilerRejectsDppOn64BitCompares() { - const uint32_t opcodes[] = {0xa2u, 0xe5u}; // v_cmp_eq_i64, v_cmp_ne_u64 - for (const auto opcode: opcodes) { - const uint32_t shader[] = { - EncodeVopc(opcode, 250u, 0u), // DPP escape in SRC0 - EncodeVop2Dpp(0u), - 0xbf810000u, - }; + const uint32_t opcodes[] = {0xa2u, 0xe5u}; // v_cmp_eq_i64, v_cmp_ne_u64 + for (const auto opcode : opcodes) { + const uint32_t shader[] = { + EncodeVopc(opcode, 250u, 0u), // DPP escape in SRC0 + EncodeVop2Dpp(0u), + 0xbf810000u, + }; - ShaderRecompiler::Decoder::Program program; - std::string error; - Check(ShaderRecompiler::Decoder::DecodeProgram(shader, program, &error), error.c_str()); - Check(program.instructions.size() == 2u, - "64-bit VOPC DPP decode did not consume its modifier word"); - const auto& compare = program.instructions.front(); - Check(compare.opcode == ShaderRecompiler::Decoder::Opcode::Unsupported, - "64-bit VOPC illegally accepted a DPP modifier"); - Check(Common::ContainsStr(compare.unsupported_reason, - "VOPC DPP modifier is not supported for opcode"), - "64-bit VOPC DPP rejection reason was not explicit"); - } + ShaderRecompiler::Decoder::Program program; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(shader, program, &error), + error.c_str()); + Check(program.instructions.size() == 2u, + "64-bit VOPC DPP decode did not consume its modifier word"); + const auto &compare = program.instructions.front(); + Check(compare.opcode == ShaderRecompiler::Decoder::Opcode::Unsupported, + "64-bit VOPC illegally accepted a DPP modifier"); + Check(Common::ContainsStr(compare.unsupported_reason, + "VOPC DPP modifier is not supported for opcode"), + "64-bit VOPC DPP rejection reason was not explicit"); + } } void TestNewShaderRecompilerIrLookupMissFailsExplicitly() { - ShaderRecompiler::Decoder::Program decoded; - ShaderRecompiler::Decoder::Instruction missing; - missing.pc = 0u; - missing.family = ShaderRecompiler::Decoder::Family::VOP1; - missing.opcode = ShaderRecompiler::Decoder::Opcode::Unknown; - decoded.instructions.push_back(missing); + ShaderRecompiler::Decoder::Program decoded; + ShaderRecompiler::Decoder::Instruction missing; + missing.pc = 0u; + missing.family = ShaderRecompiler::Decoder::Family::VOP1; + missing.opcode = ShaderRecompiler::Decoder::Opcode::Unknown; + decoded.instructions.push_back(missing); - ShaderRecompiler::CFG::Graph cfg; - ShaderRecompiler::CFG::BasicBlock block; - block.inst_end = 1u; - cfg.blocks.push_back(block); - cfg.entry_block = 0u; + ShaderRecompiler::CFG::Graph cfg; + ShaderRecompiler::CFG::BasicBlock block; + block.inst_end = 1u; + cfg.blocks.push_back(block); + cfg.entry_block = 0u; - ShaderRecompiler::IR::Program ir; - std::string error; - Check(!ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, 64u, ir, &error), - "missing decoder-to-IR mapping unexpectedly lowered as an instruction"); - Check(Common::ContainsStr(error, "no IR lowering"), - "missing decoder-to-IR mapping did not report an explicit error"); - Check(ir.blocks.empty(), "missing decoder-to-IR mapping emitted a fallback IR block"); + ShaderRecompiler::IR::Program ir; + std::string error; + Check(!ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, + 64u, ir, &error), + "missing decoder-to-IR mapping unexpectedly lowered as an instruction"); + Check(Common::ContainsStr(error, "no IR lowering"), + "missing decoder-to-IR mapping did not report an explicit error"); + Check(ir.blocks.empty(), + "missing decoder-to-IR mapping emitted a fallback IR block"); - decoded.instructions.front().family = ShaderRecompiler::Decoder::Family::MUBUF; - decoded.instructions.front().opcode = ShaderRecompiler::Decoder::Opcode::VAddNcU32; - error.clear(); - Check(!ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, 64u, ir, &error), - "memory-family opcode bypassed specialized memory lowering"); - Check(Common::ContainsStr(error, "memory-family opcode") && - Common::ContainsStr(error, "specialized IR lowering"), - "memory-family lowering bypass did not report an explicit error"); + decoded.instructions.front().family = + ShaderRecompiler::Decoder::Family::MUBUF; + decoded.instructions.front().opcode = + ShaderRecompiler::Decoder::Opcode::VAddNcU32; + error.clear(); + Check(!ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, + 64u, ir, &error), + "memory-family opcode bypassed specialized memory lowering"); + Check(Common::ContainsStr(error, "memory-family opcode") && + Common::ContainsStr(error, "specialized IR lowering"), + "memory-family lowering bypass did not report an explicit error"); } void TestNewShaderRecompilerMemoryFamilyLowering() { - const uint32_t shader[] = { - EncodeSmem0(0x00, 0, 4), - 0u, // s_load_dword s0 - EncodeSmem0(0x08, 1, 4), - 0u, // s_buffer_load_dword s1 - EncodeMubuf0(0x0c, 4), - EncodeMubuf1(0, 0, 1), // buffer_load_dword v0 - EncodeMubuf0(0x1c, 8), - EncodeMubuf1(0, 0, 1), // buffer_store_dword v0 - EncodeDs0(0x0d), - EncodeDs1(0, 0, 1), // ds_write_b32 v0, v1 - EncodeDs0(0x36), - EncodeDs1(2, 0, 1), // ds_read_b32 v2, v1 - EncodeMimg0(0x0e, 0x1), - EncodeMimg1(5, 0, 0, 1), // image_get_resinfo v5 - EncodeMimg0(0x00, 0xf), - EncodeMimg1(4, 0, 0, 1), // image_load v4 - EncodeMimg0(0x20, 0xf), - EncodeMimg1(3, 0, 0, 1), // image_sample v3 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSmem0(0x00, 0, 4), + 0u, // s_load_dword s0 + EncodeSmem0(0x08, 1, 4), + 0u, // s_buffer_load_dword s1 + EncodeMubuf0(0x0c, 4), + EncodeMubuf1(0, 0, 1), // buffer_load_dword v0 + EncodeMubuf0(0x1c, 8), + EncodeMubuf1(0, 0, 1), // buffer_store_dword v0 + EncodeDs0(0x0d), + EncodeDs1(0, 0, 1), // ds_write_b32 v0, v1 + EncodeDs0(0x36), + EncodeDs1(2, 0, 1), // ds_read_b32 v2, v1 + EncodeMimg0(0x0e, 0x1), + EncodeMimg1(5, 0, 0, 1), // image_get_resinfo v5 + EncodeMimg0(0x00, 0xf), + EncodeMimg1(4, 0, 0, 1), // image_load v4 + EncodeMimg0(0x20, 0xf), + EncodeMimg1(3, 0, 0, 1), // image_sample v3 + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); - options.read_memory = ReadZeroTestMemory; + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); + options.read_memory = ReadZeroTestMemory; - ShaderRecompiler::CompileResult result; - std::string error; - const auto compiled = ShaderRecompiler::TryRecompile(shader, options, result, &error); - Check(compiled, error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_load_dword"), - "new decoder did not decode SMEM dword load"); - Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword"), - "new decoder did not decode SMEM scalar-buffer dword load"); - Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword s1, s8"), - "SMEM scalar-buffer SBASE was not decoded as an SGPR-pair index"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dword"), - "new decoder did not decode MUBUF dword load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dword"), - "new decoder did not decode MUBUF dword store"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b32"), - "new decoder did not decode DS dword read"); - Check(Common::ContainsStr(result.decoded_dump, "image_get_resinfo"), - "new decoder did not decode MIMG resinfo query"); - Check(Common::ContainsStr(result.decoded_dump, "image_load"), - "new decoder did not decode MIMG load"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample"), - "new decoder did not decode MIMG sample"); - Check(!Common::ContainsStr(result.decoded_dump, "lowering is not implemented"), - "implemented memory decode still reports unsupported lowering"); - Check(Common::ContainsStr(result.ir_dump, "SLoadDword s0"), "SMEM load did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s1"), - "SMEM scalar-buffer load did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v0"), - "MUBUF load did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v0"), - "MUBUF store did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v0"), - "DS write did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v2"), "DS read did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ImageGetResinfo v5"), - "MIMG resinfo query did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ImageLoad v4"), "MIMG load did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "ImageSample v3"), "MIMG sample did not lower to IR"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 95), "SPIR-V binary does not contain OpImageFetch"); - Check(SpirvContainsOpcode(result.spirv, 103), - "SPIR-V binary does not contain OpImageQuerySizeLod"); - Check(SpirvContainsOpcode(result.spirv, 88), - "SPIR-V binary does not contain OpImageSampleExplicitLod"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(compiled, error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_load_dword"), + "new decoder did not decode SMEM dword load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword"), + "new decoder did not decode SMEM scalar-buffer dword load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword s1, s8"), + "SMEM scalar-buffer SBASE was not decoded as an SGPR-pair index"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dword"), + "new decoder did not decode MUBUF dword load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dword"), + "new decoder did not decode MUBUF dword store"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b32"), + "new decoder did not decode DS dword read"); + Check(Common::ContainsStr(result.decoded_dump, "image_get_resinfo"), + "new decoder did not decode MIMG resinfo query"); + Check(Common::ContainsStr(result.decoded_dump, "image_load"), + "new decoder did not decode MIMG load"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample"), + "new decoder did not decode MIMG sample"); + Check( + !Common::ContainsStr(result.decoded_dump, "lowering is not implemented"), + "implemented memory decode still reports unsupported lowering"); + Check(Common::ContainsStr(result.ir_dump, "SLoadDword s0"), + "SMEM load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s1"), + "SMEM scalar-buffer load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v0"), + "MUBUF load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v0"), + "MUBUF store did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v0"), + "DS write did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v2"), + "DS read did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageGetResinfo v5"), + "MIMG resinfo query did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v4"), + "MIMG load did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v3"), + "MIMG sample did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 95), + "SPIR-V binary does not contain OpImageFetch"); + Check(SpirvContainsOpcode(result.spirv, 103), + "SPIR-V binary does not contain OpImageQuerySizeLod"); + Check(SpirvContainsOpcode(result.spirv, 88), + "SPIR-V binary does not contain OpImageSampleExplicitLod"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageQueryLowering() { - const uint32_t shader[] = { - EncodeMimg0(0x60, 0x3), - EncodeMimg1(6, 0, 0, 1), // image_get_lod - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x60, 0x3), + EncodeMimg1(6, 0, 0, 1), // image_get_lod + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_get_lod"), - "new decoder did not decode MIMG image get-lod query"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), - "image_get_lod decode did not preserve dmask metadata"); - Check(Common::ContainsStr(result.ir_dump, "ImageGetLod v6"), - "image_get_lod did not lower to explicit query IR"); - Check(Common::ContainsStr(result.ir_dump, "data_dwords=2"), - "image_get_lod did not preserve two-component result metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=2"), - "image_get_lod did not preserve address component metadata"); - Check(SpirvContainsOpcode(result.spirv, 105), "SPIR-V binary does not contain OpImageQueryLod"); - Check(SpirvContainsOpcode(result.spirv, 80), - "SPIR-V binary does not contain coordinate composite for image_get_lod"); - Check(SpirvContainsOpcode(result.spirv, 81), - "SPIR-V binary does not contain dmask extraction for image_get_lod"); - Check(SpirvContainsOpcode(result.spirv, 124), - "SPIR-V binary does not contain result bitcast for image_get_lod"); - Check(std::find(result.spirv.begin(), result.spirv.end(), 5288u) != result.spirv.end(), - "SPIR-V binary does not request compute derivative group capability"); - Check(std::find(result.spirv.begin(), result.spirv.end(), 5289u) != result.spirv.end(), - "SPIR-V binary does not request compute derivative group execution mode"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_get_lod"), + "new decoder did not decode MIMG image get-lod query"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), + "image_get_lod decode did not preserve dmask metadata"); + Check(Common::ContainsStr(result.ir_dump, "ImageGetLod v6"), + "image_get_lod did not lower to explicit query IR"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=2"), + "image_get_lod did not preserve two-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=2"), + "image_get_lod did not preserve address component metadata"); + Check(SpirvContainsOpcode(result.spirv, 105), + "SPIR-V binary does not contain OpImageQueryLod"); + Check( + SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain coordinate composite for image_get_lod"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain dmask extraction for image_get_lod"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain result bitcast for image_get_lod"); + Check(std::find(result.spirv.begin(), result.spirv.end(), 5288u) != + result.spirv.end(), + "SPIR-V binary does not request compute derivative group capability"); + Check( + std::find(result.spirv.begin(), result.spirv.end(), 5289u) != + result.spirv.end(), + "SPIR-V binary does not request compute derivative group execution mode"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCubeSampleCoordinates() { - constexpr uint32_t MimgDimCube = 3; - const uint32_t shader[] = { - EncodeMimg0(0x20, 0xf, false, MimgDimCube), - EncodeMimg1(0, 0, 1, 0), // image_sample cube - EncodeMimg0(0x60, 0x3, false, MimgDimCube), - EncodeMimg1(8, 0, 1, 4), // image_get_lod cube - 0xbf810000u, - }; + constexpr uint32_t MimgDimCube = 3; + const uint32_t shader[] = { + EncodeMimg0(0x20, 0xf, false, MimgDimCube), + EncodeMimg1(0, 0, 1, 0), // image_sample cube + EncodeMimg0(0x60, 0x3, false, MimgDimCube), + EncodeMimg1(8, 0, 1, 4), // image_get_lod cube + 0xbf810000u, + }; - auto user_data = ImageTestUserData(Prospero::ImageType::kCube); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(Prospero::ImageType::kCube); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(result.program.info.images.size() == 1 && result.program.info.images[0].cube, - "cube descriptor identity was not preserved through compilation"); - Check(SpirvInstructionOpcodeCount(result.spirv, 131) == 4, - "cube sample/get-lod did not remove the RDNA2 S/T bias"); - Check(SpirvInstructionOpcodeCount(result.spirv, 109) == 1 && - SpirvInstructionOpcodeCount(result.spirv, 112) == 1 && - SpirvInstructionOpcodeCount(result.spirv, 194) == 1 && - SpirvInstructionOpcodeCount(result.spirv, 196) == 1 && - SpirvInstructionOpcodeCount(result.spirv, 130) == 1, - "cube sample did not repack its face ID exactly once, or get-lod used an array layer"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(result.program.info.images.size() == 1 && + result.program.info.images[0].cube, + "cube descriptor identity was not preserved through compilation"); + Check(SpirvInstructionOpcodeCount(result.spirv, 131) == 4, + "cube sample/get-lod did not remove the RDNA2 S/T bias"); + Check(SpirvInstructionOpcodeCount(result.spirv, 109) == 1 && + SpirvInstructionOpcodeCount(result.spirv, 112) == 1 && + SpirvInstructionOpcodeCount(result.spirv, 194) == 1 && + SpirvInstructionOpcodeCount(result.spirv, 196) == 1 && + SpirvInstructionOpcodeCount(result.spirv, 130) == 1, + "cube sample did not repack its face ID exactly once, or get-lod used " + "an array layer"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageSampleVariants() { - const uint32_t shader[] = { - EncodeMimg0(0x24, 0xf), - EncodeMimg1(8, 0, 0, 1), // image_sample_l - EncodeMimg0(0x25, 0xf), - EncodeMimg1(12, 0, 0, 1), // image_sample_b - EncodeMimg0(0x27, 0xf), - EncodeMimg1(16, 0, 0, 1), // image_sample_lz - EncodeMimg0(0x28, 0xf), - EncodeMimg1(20, 0, 0, 1), // image_sample_c - EncodeMimg0(0x30, 0xf), - EncodeMimg1(24, 0, 0, 1), // image_sample_o - EncodeMimg0(0x22, 0xf), - EncodeMimg1(28, 0, 0, 1), // image_sample_d - EncodeMimg0(0x2f, 0x1), - EncodeMimg1(32, 0, 0, 1), // image_sample_c_lz - EncodeMimg0(0x38, 0x1), - EncodeMimg1(36, 0, 0, 1), // image_sample_c_o - EncodeMimg0(0x2a, 0x1), - EncodeMimg1(40, 0, 0, 1), // image_sample_c_d - EncodeMimg0(0x20, 0x7), - EncodeMimg1(48, 0, 0, 1, true), // image_sample with A16 bit - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x24, 0xf), + EncodeMimg1(8, 0, 0, 1), // image_sample_l + EncodeMimg0(0x25, 0xf), + EncodeMimg1(12, 0, 0, 1), // image_sample_b + EncodeMimg0(0x27, 0xf), + EncodeMimg1(16, 0, 0, 1), // image_sample_lz + EncodeMimg0(0x28, 0xf), + EncodeMimg1(20, 0, 0, 1), // image_sample_c + EncodeMimg0(0x30, 0xf), + EncodeMimg1(24, 0, 0, 1), // image_sample_o + EncodeMimg0(0x22, 0xf), + EncodeMimg1(28, 0, 0, 1), // image_sample_d + EncodeMimg0(0x2f, 0x1), + EncodeMimg1(32, 0, 0, 1), // image_sample_c_lz + EncodeMimg0(0x38, 0x1), + EncodeMimg1(36, 0, 0, 1), // image_sample_c_o + EncodeMimg0(0x2a, 0x1), + EncodeMimg1(40, 0, 0, 1), // image_sample_c_d + EncodeMimg0(0x20, 0x7), + EncodeMimg1(48, 0, 0, 1, true), // image_sample with A16 bit + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), - "new decoder did not decode IMAGE_SAMPLE_L through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_b"), - "new decoder did not decode IMAGE_SAMPLE_B through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_lz"), - "new decoder did not decode IMAGE_SAMPLE_LZ through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_c"), - "new decoder did not decode IMAGE_SAMPLE_C through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_o"), - "new decoder did not decode IMAGE_SAMPLE_O through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_d"), - "new decoder did not decode IMAGE_SAMPLE_D through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_lz"), - "new decoder did not decode IMAGE_SAMPLE_C_LZ through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_o"), - "new decoder did not decode IMAGE_SAMPLE_C_O through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_d"), - "new decoder did not decode IMAGE_SAMPLE_C_D through shared MIMG path"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=a16"), - "IMAGE_SAMPLE with MIMG A16 bit did not expose A16 sample flag"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=lod"), - "IMAGE_SAMPLE_L did not expose lod sample flag"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias"), - "IMAGE_SAMPLE_B did not expose bias sample flag"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=level_zero"), - "IMAGE_SAMPLE_LZ did not expose level-zero sample flag"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare"), - "IMAGE_SAMPLE_C did not expose compare sample flag"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=offset addr_components=3"), - "IMAGE_SAMPLE_O did not expose offset sample flag/address width"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=derivative addr_components=6"), - "IMAGE_SAMPLE_D did not expose derivative sample flag/address width"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare|level_zero"), - "IMAGE_SAMPLE_C_LZ did not expose compare+level-zero flags"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare|offset addr_components=4"), - "IMAGE_SAMPLE_C_O did not expose compare+offset flags/address width"); - Check(Common::ContainsStr(result.decoded_dump, - "sample_flags=derivative|compare addr_components=7"), - "IMAGE_SAMPLE_C_D did not expose compare+derivative address width"); - Check(Common::ContainsStr(result.ir_dump, "ImageSample v8"), - "IMAGE_SAMPLE_L did not lower to shared IR ImageSample"); - Check(Common::ContainsStr(result.ir_dump, "ImageSample v28"), - "IMAGE_SAMPLE_D did not lower to shared IR ImageSample"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x4"), - "derivative sample flag did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=6"), - "derivative sample address width did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=7"), - "compare+derivative sample address width did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), - "MIMG A16 bit did not survive into IR memory metadata"); - Check(SpirvContainsOpcode(result.spirv, 88), - "SPIR-V binary does not contain shared OpImageSampleExplicitLod emission"); - Check(SpirvContainsOpcode(result.spirv, 90), - "SPIR-V binary does not contain shared OpImageSampleDrefExplicitLod emission"); - Check(SpirvContainsOpcode(result.spirv, 80), - "SPIR-V binary does not contain coordinate/gradient composite construction"); - Check(SpirvContainsExtInst(result.spirv, 62), - "SPIR-V binary does not unpack A16 image address halves"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), + "new decoder did not decode IMAGE_SAMPLE_L through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_b"), + "new decoder did not decode IMAGE_SAMPLE_B through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_lz"), + "new decoder did not decode IMAGE_SAMPLE_LZ through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c"), + "new decoder did not decode IMAGE_SAMPLE_C through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_o"), + "new decoder did not decode IMAGE_SAMPLE_O through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_d"), + "new decoder did not decode IMAGE_SAMPLE_D through shared MIMG path"); + Check( + Common::ContainsStr(result.decoded_dump, "image_sample_c_lz"), + "new decoder did not decode IMAGE_SAMPLE_C_LZ through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_o"), + "new decoder did not decode IMAGE_SAMPLE_C_O through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_d"), + "new decoder did not decode IMAGE_SAMPLE_C_D through shared MIMG path"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=a16"), + "IMAGE_SAMPLE with MIMG A16 bit did not expose A16 sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=lod"), + "IMAGE_SAMPLE_L did not expose lod sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias"), + "IMAGE_SAMPLE_B did not expose bias sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=level_zero"), + "IMAGE_SAMPLE_LZ did not expose level-zero sample flag"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare"), + "IMAGE_SAMPLE_C did not expose compare sample flag"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=offset addr_components=3"), + "IMAGE_SAMPLE_O did not expose offset sample flag/address width"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=derivative addr_components=6"), + "IMAGE_SAMPLE_D did not expose derivative sample flag/address width"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|level_zero"), + "IMAGE_SAMPLE_C_LZ did not expose compare+level-zero flags"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|offset addr_components=4"), + "IMAGE_SAMPLE_C_O did not expose compare+offset flags/address width"); + Check( + Common::ContainsStr(result.decoded_dump, + "sample_flags=derivative|compare addr_components=7"), + "IMAGE_SAMPLE_C_D did not expose compare+derivative address width"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v8"), + "IMAGE_SAMPLE_L did not lower to shared IR ImageSample"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v28"), + "IMAGE_SAMPLE_D did not lower to shared IR ImageSample"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x4"), + "derivative sample flag did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=6"), + "derivative sample address width did not survive into IR memory " + "metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=7"), + "compare+derivative sample address width did not survive into IR " + "memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), + "MIMG A16 bit did not survive into IR memory metadata"); + Check(SpirvContainsOpcode(result.spirv, 88), + "SPIR-V binary does not contain shared OpImageSampleExplicitLod " + "emission"); + Check(SpirvContainsOpcode(result.spirv, 90), + "SPIR-V binary does not contain shared OpImageSampleDrefExplicitLod " + "emission"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain coordinate/gradient composite " + "construction"); + Check(SpirvContainsExtInst(result.spirv, 62), + "SPIR-V binary does not unpack A16 image address halves"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageSampleA16SamplerCoords() { - constexpr uint32_t MimgDim3D = 2; + constexpr uint32_t MimgDim3D = 2; - const uint32_t shader[] = { - EncodeMimg0(0x20, 0x7, false, MimgDim3D), - EncodeMimg1(8, 0, 0, 4, true), // image_sample, A16 bit, 3D xyz coords - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x20, 0x7, false, MimgDim3D), + EncodeMimg1(8, 0, 0, 4, true), // image_sample, A16 bit, 3D xyz coords + 0xbf810000u, + }; - auto user_data = ImageTestUserData(Prospero::ImageType::kColor3D); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(Prospero::ImageType::kColor3D); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - const auto compiled = ShaderRecompiler::TryRecompile(shader, options, result, &error); - Check(compiled, error.c_str()); - Check( - Common::ContainsStr(result.decoded_dump, "image_dim=3d sample_flags=a16 addr_components=3"), - "3D IMAGE_SAMPLE with MIMG A16 bit did not decode as three A16 sampler coords"); - Check(Common::ContainsStr(result.ir_dump, "ImageSample v8"), - "3D A16 IMAGE_SAMPLE did not lower to sample IR"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), - "3D A16 IMAGE_SAMPLE did not preserve A16 memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_dim=3d"), - "3D A16 IMAGE_SAMPLE did not preserve image dimension metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=3"), - "3D A16 IMAGE_SAMPLE did not preserve three logical address components"); - Check(SpirvExtInstCount(result.spirv, 62) == 3, - "sampler A16 xyz coordinates should be converted from three packed f16 components"); - Check(SpirvContainsOpcode(result.spirv, 194), - "sampler A16 high-half coordinate extraction is missing"); - Check(SpirvContainsOpcode(result.spirv, 199), - "sampler A16 low-half coordinate masking is missing"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(compiled, error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, + "image_dim=3d sample_flags=a16 addr_components=3"), + "3D IMAGE_SAMPLE with MIMG A16 bit did not decode as three A16 sampler " + "coords"); + Check(Common::ContainsStr(result.ir_dump, "ImageSample v8"), + "3D A16 IMAGE_SAMPLE did not lower to sample IR"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), + "3D A16 IMAGE_SAMPLE did not preserve A16 memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=3d"), + "3D A16 IMAGE_SAMPLE did not preserve image dimension metadata"); + Check( + Common::ContainsStr(result.ir_dump, "image_addr=3"), + "3D A16 IMAGE_SAMPLE did not preserve three logical address components"); + Check(SpirvExtInstCount(result.spirv, 62) == 3, + "sampler A16 xyz coordinates should be converted from three packed f16 " + "components"); + Check(SpirvContainsOpcode(result.spirv, 194), + "sampler A16 high-half coordinate extraction is missing"); + Check(SpirvContainsOpcode(result.spirv, 199), + "sampler A16 low-half coordinate masking is missing"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageSampleOpcodeAliases() { - const uint32_t shader[] = { - 0xf0800109u, - 0x00c00602u, // observed image_sample_a v6, v2, s0, s24 - EncodeMimg0(0xa5, 0xf), - EncodeMimg1(12, 0, 0, 4), // image_sample_b_a - EncodeMimg0(0xa8, 0x1), - EncodeMimg1(16, 0, 0, 4), // image_sample_c_a - EncodeMimg0(0xad, 0x1), - EncodeMimg1(20, 0, 0, 4), // image_sample_c_b_a - 0xbf810000u, - }; + const uint32_t shader[] = { + 0xf0800109u, + 0x00c00602u, // observed image_sample_a v6, v2, s0, s24 + EncodeMimg0(0xa5, 0xf), + EncodeMimg1(12, 0, 0, 4), // image_sample_b_a + EncodeMimg0(0xa8, 0x1), + EncodeMimg1(16, 0, 0, 4), // image_sample_c_a + EncodeMimg0(0xad, 0x1), + EncodeMimg1(20, 0, 0, 4), // image_sample_c_b_a + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_a"), - "MIMG opcode 0xa0 should decode as image_sample_a alias"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_b_a"), - "MIMG opcode 0xa5 should decode as image_sample_b_a alias"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_a"), - "MIMG opcode 0xa8 should decode as image_sample_c_a alias"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_b_a"), - "MIMG opcode 0xad should decode as image_sample_c_b_a alias"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=none"), - "opcode 0xa0 alias should use normal 32-bit sample coordinates"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias|compare"), - "compare+bias opcode alias did not expose expected sample flags"); - Check(!Common::ContainsStr(result.decoded_dump, "a16"), - "A16 must come from MIMG bit 62, not from the opcode alias"); - Check(!SpirvContainsExtInst(result.spirv, 62), - "opcode aliases without bit 62 must not unpack sampled f16 address halves"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_a"), + "MIMG opcode 0xa0 should decode as image_sample_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_b_a"), + "MIMG opcode 0xa5 should decode as image_sample_b_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_a"), + "MIMG opcode 0xa8 should decode as image_sample_c_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_c_b_a"), + "MIMG opcode 0xad should decode as image_sample_c_b_a alias"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=none"), + "opcode 0xa0 alias should use normal 32-bit sample coordinates"); + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias|compare"), + "compare+bias opcode alias did not expose expected sample flags"); + Check(!Common::ContainsStr(result.decoded_dump, "a16"), + "A16 must come from MIMG bit 62, not from the opcode alias"); + Check(!SpirvContainsExtInst(result.spirv, 62), + "opcode aliases without bit 62 must not unpack sampled f16 address " + "halves"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageSampleA16ExceptionComponents() { - auto compile = [](uint32_t opcode, uint32_t dmask, uint32_t vdata) { - const uint32_t shader[] = { - EncodeMimg0(opcode, dmask), - EncodeMimg1(vdata, 0, 0, 4, true), - 0xbf810000u, - }; + auto compile = [](uint32_t opcode, uint32_t dmask, uint32_t vdata) { + const uint32_t shader[] = { + EncodeMimg0(opcode, dmask), + EncodeMimg1(vdata, 0, 0, 4, true), + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - return result; - }; + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + return result; + }; - { - const auto result = compile(0x28, 0x1, 12); // image_sample_c with A16 bit - Check( - Common::ContainsStr(result.decoded_dump, "sample_flags=compare|a16 addr_components=3"), - "A16 IMAGE_SAMPLE_C did not preserve compare+A16 metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x88"), - "A16 IMAGE_SAMPLE_C did not preserve compare+A16 IR flags"); - Check(SpirvInstructionOpcodeCount(result.spirv, 90) == 1, - "A16 IMAGE_SAMPLE_C should emit one dref sample"); - Check(SpirvExtInstCount(result.spirv, 62) == 2, - "PCF reference must remain 32-bit while only xy sampler coords use A16"); - } + { + const auto result = compile(0x28, 0x1, 12); // image_sample_c with A16 bit + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|a16 addr_components=3"), + "A16 IMAGE_SAMPLE_C did not preserve compare+A16 metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x88"), + "A16 IMAGE_SAMPLE_C did not preserve compare+A16 IR flags"); + Check(SpirvInstructionOpcodeCount(result.spirv, 90) == 1, + "A16 IMAGE_SAMPLE_C should emit one dref sample"); + Check(SpirvExtInstCount(result.spirv, 62) == 2, + "PCF reference must remain 32-bit while only xy sampler coords use " + "A16"); + } - { - const auto result = compile(0x30, 0xf, 16); // image_sample_o with A16 bit - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=offset|a16 addr_components=3"), - "A16 IMAGE_SAMPLE_O did not preserve offset+A16 metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x90"), - "A16 IMAGE_SAMPLE_O did not preserve offset+A16 IR flags"); - Check(SpirvContainsOpcode(result.spirv, 202), - "texel offset should still be decoded from its packed 6-bit fields"); - Check(SpirvExtInstCount(result.spirv, 62) == 2, - "texel offset must remain 32-bit while only xy sampler coords use A16"); - } + { + const auto result = compile(0x30, 0xf, 16); // image_sample_o with A16 bit + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=offset|a16 addr_components=3"), + "A16 IMAGE_SAMPLE_O did not preserve offset+A16 metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x90"), + "A16 IMAGE_SAMPLE_O did not preserve offset+A16 IR flags"); + Check(SpirvContainsOpcode(result.spirv, 202), + "texel offset should still be decoded from its packed 6-bit fields"); + Check( + SpirvExtInstCount(result.spirv, 62) == 2, + "texel offset must remain 32-bit while only xy sampler coords use A16"); + } } void TestNewShaderRecompilerImageSampleA16Rdna2AddressOrder() { - namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; - namespace Emitter = Libs::Graphics::ShaderRecompiler::Spirv::Emitter; - namespace IR = Libs::Graphics::ShaderRecompiler::IR; + namespace Decoder = Libs::Graphics::ShaderRecompiler::Decoder; + namespace Emitter = Libs::Graphics::ShaderRecompiler::Spirv::Emitter; + namespace IR = Libs::Graphics::ShaderRecompiler::IR; - auto make_inst = [](uint32_t flags) { - IR::Instruction inst {}; - inst.src[0] = Emitter::MakeRegisterOperand(IR::RegisterFile::Vector, 20); - inst.memory.image_sample_flags = flags | Decoder::ImageSampleFlagA16; - inst.memory.image_address_components = 5; - return inst; - }; - auto check_vgpr = [](const IR::Operand& operand, uint32_t expected, const char* message) { - Check(operand.kind == IR::OperandKind::Register && - operand.reg.file == IR::RegisterFile::Vector && operand.reg.index == expected, - message); - }; + auto make_inst = [](uint32_t flags) { + IR::Instruction inst{}; + inst.src[0] = Emitter::MakeRegisterOperand(IR::RegisterFile::Vector, 20); + inst.memory.image_sample_flags = flags | Decoder::ImageSampleFlagA16; + inst.memory.image_address_components = 5; + return inst; + }; + auto check_vgpr = [](const IR::Operand &operand, uint32_t expected, + const char *message) { + Check(operand.kind == IR::OperandKind::Register && + operand.reg.file == IR::RegisterFile::Vector && + operand.reg.index == expected, + message); + }; - { - auto inst = make_inst(Decoder::ImageSampleFlagBias | Decoder::ImageSampleFlagCompare); - const auto layout = Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); - Check(layout.bias == 0u, "RDNA2 sample_c_b A16 bias must precede PCF reference"); - Check(layout.dref == 1u, "RDNA2 sample_c_b A16 PCF reference must follow bias"); - Check(layout.coord == 2u, "RDNA2 sample_c_b A16 body must follow bias and PCF"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 0u, - "A16 bias should use the first packed half"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 2u, - "32-bit PCF reference must align to the next VGPR after A16 bias"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 4u, - "A16 coordinates must start after the aligned PCF reference"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 20, - "A16 sample_c_b bias should read v20"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 21, - "A16 sample_c_b PCF reference should read v21"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), 22, - "A16 sample_c_b x/y coordinates should start in v22"); - } + { + auto inst = make_inst(Decoder::ImageSampleFlagBias | + Decoder::ImageSampleFlagCompare); + const auto layout = + Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); + Check(layout.bias == 0u, + "RDNA2 sample_c_b A16 bias must precede PCF reference"); + Check(layout.dref == 1u, + "RDNA2 sample_c_b A16 PCF reference must follow bias"); + Check(layout.coord == 2u, + "RDNA2 sample_c_b A16 body must follow bias and PCF"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 0u, + "A16 bias should use the first packed half"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 2u, + "32-bit PCF reference must align to the next VGPR after A16 bias"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 4u, + "A16 coordinates must start after the aligned PCF reference"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 20, + "A16 sample_c_b bias should read v20"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 21, + "A16 sample_c_b PCF reference should read v21"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), + 22, "A16 sample_c_b x/y coordinates should start in v22"); + } - { - auto inst = make_inst(Decoder::ImageSampleFlagOffset | Decoder::ImageSampleFlagBias | - Decoder::ImageSampleFlagCompare); - const auto layout = Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); - Check(layout.offset == 0u && layout.bias == 1u && layout.dref == 2u && layout.coord == 3u, - "RDNA2 sample_c_b_o order must be offset, bias, PCF, body"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.offset) == 0u, - "texel offset should stay a packed 32-bit VGPR"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 2u, - "A16 bias should follow the 32-bit texel offset"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 4u, - "PCF reference should align after A16 bias"); - Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 6u, - "A16 coordinates should start after offset, bias, and PCF"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.offset), 20, - "A16 sample_c_b_o offset should read v20"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 21, - "A16 sample_c_b_o bias should read v21"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 22, - "A16 sample_c_b_o PCF reference should read v22"); - check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), 23, - "A16 sample_c_b_o x/y coordinates should start in v23"); - } + { + auto inst = make_inst(Decoder::ImageSampleFlagOffset | + Decoder::ImageSampleFlagBias | + Decoder::ImageSampleFlagCompare); + const auto layout = + Emitter::MakeImageSampleLayout(inst, Emitter::ImageViewKind::Dim2D); + Check(layout.offset == 0u && layout.bias == 1u && layout.dref == 2u && + layout.coord == 3u, + "RDNA2 sample_c_b_o order must be offset, bias, PCF, body"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.offset) == 0u, + "texel offset should stay a packed 32-bit VGPR"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.bias) == 2u, + "A16 bias should follow the 32-bit texel offset"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.dref) == 4u, + "PCF reference should align after A16 bias"); + Check(Emitter::ImageAddressHalfComponent(inst, layout.coord) == 6u, + "A16 coordinates should start after offset, bias, and PCF"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.offset), + 20, "A16 sample_c_b_o offset should read v20"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.bias), 21, + "A16 sample_c_b_o bias should read v21"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.dref), 22, + "A16 sample_c_b_o PCF reference should read v22"); + check_vgpr(Emitter::ImageAddressOperand(inst, inst.src[0], layout.coord), + 23, "A16 sample_c_b_o x/y coordinates should start in v23"); + } } void TestNewShaderRecompilerImageLoadA16UintCoords() { - const uint32_t shader[] = { - EncodeMimg0(0x00, 0x3), - EncodeMimg1(20, 0, 0, 4, true), // image_load, A16 bit - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x00, 0x3), + EncodeMimg1(20, 0, 0, 4, true), // image_load, A16 bit + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_load"), "A16 IMAGE_LOAD did not decode"); - Check(Common::ContainsStr(result.ir_dump, "ImageLoad v20"), - "A16 IMAGE_LOAD did not lower to image-load IR"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), - "A16 IMAGE_LOAD did not preserve A16 memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=2"), - "A16 IMAGE_LOAD did not preserve logical address component count"); - Check(!SpirvContainsExtInst(result.spirv, 62), - "image ops without sampler use u16 A16 addresses and must not unpack f16"); - Check(SpirvContainsOpcode(result.spirv, 194), - "u16 A16 high-half coordinate extraction is missing"); - Check(SpirvContainsOpcode(result.spirv, 199), "u16 A16 low-half coordinate masking is missing"); - Check(SpirvContainsOpcode(result.spirv, 95), "A16 IMAGE_LOAD did not emit an image fetch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_load"), + "A16 IMAGE_LOAD did not decode"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v20"), + "A16 IMAGE_LOAD did not lower to image-load IR"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x80"), + "A16 IMAGE_LOAD did not preserve A16 memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=2"), + "A16 IMAGE_LOAD did not preserve logical address component count"); + Check(!SpirvContainsExtInst(result.spirv, 62), + "image ops without sampler use u16 A16 addresses and must not unpack " + "f16"); + Check(SpirvContainsOpcode(result.spirv, 194), + "u16 A16 high-half coordinate extraction is missing"); + Check(SpirvContainsOpcode(result.spirv, 199), + "u16 A16 low-half coordinate masking is missing"); + Check(SpirvContainsOpcode(result.spirv, 95), + "A16 IMAGE_LOAD did not emit an image fetch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerPixelImageSampleLodSelection() { - constexpr uint32_t OpImageSampleImplicitLod = 87; - constexpr uint32_t OpImageSampleExplicitLod = 88; - constexpr uint32_t OpImageSampleDrefImplicitLod = 89; - constexpr uint32_t OpImageSampleDrefExplicitLod = 90; + constexpr uint32_t OpImageSampleImplicitLod = 87; + constexpr uint32_t OpImageSampleExplicitLod = 88; + constexpr uint32_t OpImageSampleDrefImplicitLod = 89; + constexpr uint32_t OpImageSampleDrefExplicitLod = 90; - auto compile = [](uint32_t opcode, uint32_t dmask) { - const uint32_t shader[] = { - EncodeMimg0(opcode, dmask), - EncodeMimg1(0, 0, 0, 4), - 0xbf810000u, - }; + auto compile = [](uint32_t opcode, uint32_t dmask) { + const uint32_t shader[] = { + EncodeMimg0(opcode, dmask), + EncodeMimg1(0, 0, 0, 4), + EncodeExp0(0x00, 0xf, true, false, true), + EncodeExp1(0, 1, 2, 3), + 0xbf810000u, + }; - auto ps_info = RegressionPixelInputInfo(); + auto ps_info = RegressionPixelInputInfo(); - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.pixel_input_info = &ps_info; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - return result; - }; + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + return result; + }; - { - const auto result = compile(0x20, 0xf); // image_sample - Check(Common::ContainsStr(result.decoded_dump, "image_sample"), - "plain pixel IMAGE_SAMPLE did not decode"); - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleImplicitLod) == 1, - "plain pixel IMAGE_SAMPLE must use OpImageSampleImplicitLod"); - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == 0, - "plain pixel IMAGE_SAMPLE unexpectedly used explicit lod"); - } - { - const auto result = compile(0x25, 0xf); // image_sample_b - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias"), - "pixel IMAGE_SAMPLE_B did not preserve bias metadata"); - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleImplicitLod) == 1, - "pixel IMAGE_SAMPLE_B must use implicit lod with a bias operand"); - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == 0, - "pixel IMAGE_SAMPLE_B unexpectedly used explicit lod"); - } - { - const auto result = compile(0x27, 0xf); // image_sample_lz - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == 1, - "pixel IMAGE_SAMPLE_LZ must use explicit lod"); - } - { - const auto result = compile(0x28, 0x1); // image_sample_c - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleDrefImplicitLod) == 1, - "pixel IMAGE_SAMPLE_C must use OpImageSampleDrefImplicitLod"); - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleDrefExplicitLod) == 0, - "pixel IMAGE_SAMPLE_C unexpectedly used explicit dref lod"); - } - { - const auto result = compile(0x2f, 0x1); // image_sample_c_lz - Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleDrefExplicitLod) == 1, - "pixel IMAGE_SAMPLE_C_LZ must use explicit dref lod"); - } + { + const auto result = compile(0x20, 0xf); // image_sample + Check(Common::ContainsStr(result.decoded_dump, "image_sample"), + "plain pixel IMAGE_SAMPLE did not decode"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleImplicitLod) == + 1, + "plain pixel IMAGE_SAMPLE must use OpImageSampleImplicitLod"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == + 0, + "plain pixel IMAGE_SAMPLE unexpectedly used explicit lod"); + Check(SpirvInstructionOpcodeCount(result.spirv, 245) == 0, + "plain pixel IMAGE_SAMPLE retained an EXEC-generated OpPhi"); + } + { + const auto result = compile(0x25, 0xf); // image_sample_b + Check(Common::ContainsStr(result.decoded_dump, "sample_flags=bias"), + "pixel IMAGE_SAMPLE_B did not preserve bias metadata"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleImplicitLod) == + 1, + "pixel IMAGE_SAMPLE_B must use implicit lod with a bias operand"); + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == + 0, + "pixel IMAGE_SAMPLE_B unexpectedly used explicit lod"); + } + { + const auto result = compile(0x27, 0xf); // image_sample_lz + Check(SpirvInstructionOpcodeCount(result.spirv, OpImageSampleExplicitLod) == + 1, + "pixel IMAGE_SAMPLE_LZ must use explicit lod"); + } + { + const auto result = compile(0x28, 0x1); // image_sample_c + Check(SpirvInstructionOpcodeCount(result.spirv, + OpImageSampleDrefImplicitLod) == 1, + "pixel IMAGE_SAMPLE_C must use OpImageSampleDrefImplicitLod"); + Check(SpirvInstructionOpcodeCount(result.spirv, + OpImageSampleDrefExplicitLod) == 0, + "pixel IMAGE_SAMPLE_C unexpectedly used explicit dref lod"); + } + { + const auto result = compile(0x2f, 0x1); // image_sample_c_lz + Check(SpirvInstructionOpcodeCount(result.spirv, + OpImageSampleDrefExplicitLod) == 1, + "pixel IMAGE_SAMPLE_C_LZ must use explicit dref lod"); + } } void TestNewShaderRecompilerImageViewDimensions() { - constexpr uint32_t MimgDim1D = 0; - constexpr uint32_t MimgDim3D = 2; - constexpr uint32_t MimgDim1DArray = 4; - constexpr uint32_t MimgDim2DArray = 5; - constexpr uint32_t SpirvDim1D = 0; - constexpr uint32_t SpirvDim2D = 1; - constexpr uint32_t SpirvDim3D = 2; + constexpr uint32_t MimgDim1D = 0; + constexpr uint32_t MimgDim3D = 2; + constexpr uint32_t MimgDim1DArray = 4; + constexpr uint32_t MimgDim2DArray = 5; + constexpr uint32_t SpirvDim1D = 0; + constexpr uint32_t SpirvDim2D = 1; + constexpr uint32_t SpirvDim3D = 2; - const uint32_t shader[] = { - EncodeMimg0(0x20, 0xf, false, MimgDim2DArray), - EncodeMimg1(0, 0, 1, 0), // image_sample 2D array - EncodeMimg0(0x24, 0xf, false, MimgDim2DArray), - EncodeMimg1(4, 1, 1, 4), // image_sample_l 2D array - EncodeMimg0(0x20, 0xf, false, MimgDim3D), - EncodeMimg1(8, 2, 1, 8), // image_sample 3D - EncodeMimg0(0x01, 0xf, false, MimgDim2DArray), - EncodeMimg1(12, 3, 0, 12), // image_load_mip 2D array - EncodeMimg0(0x20, 0xf, false, MimgDim1D), - EncodeMimg1(16, 4, 1, 16), // image_sample 1D - EncodeMimg0(0x20, 0xf, false, MimgDim1DArray), - EncodeMimg1(20, 5, 1, 20), // image_sample 1D array - EncodeMimg0(0x22, 0xf, false, MimgDim1D), - EncodeMimg1(24, 6, 1, 24), // image_sample_d 1D - EncodeMimg0(0x01, 0xf, false, MimgDim1D), - EncodeMimg1(28, 7, 0, 28), // image_load_mip 1D - EncodeMimg0(0x0e, 0xf, false, MimgDim1D), - EncodeMimg1(32, 8, 0, 32), // image_get_resinfo 1D - EncodeMimg0(0x00, 0xf, false, MimgDim1D), - EncodeMimg1(36, 9, 0, 36), // integer image_load 1D - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x20, 0xf, false, MimgDim2DArray), + EncodeMimg1(0, 0, 1, 0), // image_sample 2D array + EncodeMimg0(0x24, 0xf, false, MimgDim2DArray), + EncodeMimg1(4, 1, 1, 4), // image_sample_l 2D array + EncodeMimg0(0x20, 0xf, false, MimgDim3D), + EncodeMimg1(8, 2, 1, 8), // image_sample 3D + EncodeMimg0(0x01, 0xf, false, MimgDim2DArray), + EncodeMimg1(12, 3, 0, 12), // image_load_mip 2D array + EncodeMimg0(0x20, 0xf, false, MimgDim1D), + EncodeMimg1(16, 4, 1, 16), // image_sample 1D + EncodeMimg0(0x20, 0xf, false, MimgDim1DArray), + EncodeMimg1(20, 5, 1, 20), // image_sample 1D array + EncodeMimg0(0x22, 0xf, false, MimgDim1D), + EncodeMimg1(24, 6, 1, 24), // image_sample_d 1D + EncodeMimg0(0x01, 0xf, false, MimgDim1D), + EncodeMimg1(28, 7, 0, 28), // image_load_mip 1D + EncodeMimg0(0x0e, 0xf, false, MimgDim1D), + EncodeMimg1(32, 8, 0, 32), // image_get_resinfo 1D + EncodeMimg0(0x00, 0xf, false, MimgDim1D), + EncodeMimg1(36, 9, 0, 36), // integer image_load 1D + 0xbf810000u, + }; - auto ps_info = RegressionPixelInputInfo(); - auto user_data = ImageTestUserData(); - SetImageTestType(&user_data, 0, Prospero::ImageType::kColor2DArray); - SetImageTestType(&user_data, 1, Prospero::ImageType::kColor2DArray); - SetImageTestType(&user_data, 2, Prospero::ImageType::kColor3D); - SetImageTestType(&user_data, 3, Prospero::ImageType::kColor2DArray); - SetImageTestType(&user_data, 4, Prospero::ImageType::kColor1D); - SetImageTestType(&user_data, 5, Prospero::ImageType::kColor1DArray); - SetImageTestType(&user_data, 6, Prospero::ImageType::kColor1D); - SetImageTestType(&user_data, 7, Prospero::ImageType::kColor1D); - SetImageTestType(&user_data, 8, Prospero::ImageType::kColor1D); - SetImageTestType(&user_data, 9, Prospero::ImageType::kColor1D); - SetImageTestFormat(&user_data, 9, Prospero::BufferFormat::k32UInt); + auto ps_info = RegressionPixelInputInfo(); + auto user_data = ImageTestUserData(); + SetImageTestType(&user_data, 0, Prospero::ImageType::kColor2DArray); + SetImageTestType(&user_data, 1, Prospero::ImageType::kColor2DArray); + SetImageTestType(&user_data, 2, Prospero::ImageType::kColor3D); + SetImageTestType(&user_data, 3, Prospero::ImageType::kColor2DArray); + SetImageTestType(&user_data, 4, Prospero::ImageType::kColor1D); + SetImageTestType(&user_data, 5, Prospero::ImageType::kColor1DArray); + SetImageTestType(&user_data, 6, Prospero::ImageType::kColor1D); + SetImageTestType(&user_data, 7, Prospero::ImageType::kColor1D); + SetImageTestType(&user_data, 8, Prospero::ImageType::kColor1D); + SetImageTestType(&user_data, 9, Prospero::ImageType::kColor1D); + SetImageTestFormat(&user_data, 9, Prospero::BufferFormat::k32UInt); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.pixel_input_info = &ps_info; - options.dump_ir = true; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - const auto compiled = ShaderRecompiler::TryRecompile(shader, options, result, &error); - Check(compiled, error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_array"), - "MIMG DIM did not decode 2D-array image view"); - Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), - "MIMG DIM did not decode 3D image view"); - Check(Common::ContainsStr(result.decoded_dump, - "image_dim=1d sample_flags=none addr_components=1"), - "1D sample did not preserve its scalar coordinate"); - Check(Common::ContainsStr(result.decoded_dump, - "image_dim=1d_array sample_flags=none addr_components=2"), - "1D-array sample did not preserve coordinate plus layer"); - Check(Common::ContainsStr(result.decoded_dump, - "image_dim=1d sample_flags=derivative addr_components=3"), - "1D derivative sample did not preserve scalar gradients"); - Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), - "2D-array LOD sample did not decode"); - Check(Common::ContainsStr(result.decoded_dump, - "image_dim=2d_array sample_flags=lod addr_components=4"), - "2D-array LOD sample did not include slice plus LOD address components"); - Check(Common::ContainsStr(result.ir_dump, "image_dim=2d_array"), - "2D-array image view did not survive into IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_dim=3d"), - "3D image view did not survive into IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_dim=1d"), - "1D image view did not survive into IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_dim=1d_array"), - "1D-array image view did not survive into IR metadata"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 0, 1), - "SPIR-V binary does not contain sampled 1D image type"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 1, 1), - "SPIR-V binary does not contain sampled 1D-array image type"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 1, 1), - "SPIR-V binary does not contain sampled 2D-array image type"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim3D, 0, 1), - "SPIR-V binary does not contain sampled 3D image type"); - Check(SpirvContainsCapability(result.spirv, 43), "SPIR-V binary does not request Sampled1D"); - Check(SpirvContainsCapability(result.spirv, 44), "SPIR-V binary does not request Image1D"); - Check(SpirvContainsOpcode(result.spirv, 95), - "SPIR-V binary does not contain array image fetch"); - CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(source, " Grad "), - "1D derivative sample did not emit scalar SPIR-V gradients"); - Check(Common::ContainsStr(source, "OpImageQuerySizeLod"), - "1D resource query did not emit a scalar size query"); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "sampled_uint_1d"), - "integer 1D load did not access the uint 1D descriptor binding"); + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(compiled, error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_array"), + "MIMG DIM did not decode 2D-array image view"); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), + "MIMG DIM did not decode 3D image view"); + Check(Common::ContainsStr(result.decoded_dump, + "image_dim=1d sample_flags=none addr_components=1"), + "1D sample did not preserve its scalar coordinate"); + Check(Common::ContainsStr( + result.decoded_dump, + "image_dim=1d_array sample_flags=none addr_components=2"), + "1D-array sample did not preserve coordinate plus layer"); + Check(Common::ContainsStr( + result.decoded_dump, + "image_dim=1d sample_flags=derivative addr_components=3"), + "1D derivative sample did not preserve scalar gradients"); + Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), + "2D-array LOD sample did not decode"); + Check( + Common::ContainsStr( + result.decoded_dump, + "image_dim=2d_array sample_flags=lod addr_components=4"), + "2D-array LOD sample did not include slice plus LOD address components"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=2d_array"), + "2D-array image view did not survive into IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=3d"), + "3D image view did not survive into IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=1d"), + "1D image view did not survive into IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=1d_array"), + "1D-array image view did not survive into IR metadata"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 0, 1), + "SPIR-V binary does not contain sampled 1D image type"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 1, 1), + "SPIR-V binary does not contain sampled 1D-array image type"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 1, 1), + "SPIR-V binary does not contain sampled 2D-array image type"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim3D, 0, 1), + "SPIR-V binary does not contain sampled 3D image type"); + Check(SpirvContainsCapability(result.spirv, 43), + "SPIR-V binary does not request Sampled1D"); + Check(SpirvContainsCapability(result.spirv, 44), + "SPIR-V binary does not request Image1D"); + Check(SpirvContainsOpcode(result.spirv, 95), + "SPIR-V binary does not contain array image fetch"); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, " Grad "), + "1D derivative sample did not emit scalar SPIR-V gradients"); + Check(Common::ContainsStr(source, "OpImageQuerySizeLod"), + "1D resource query did not emit a scalar size query"); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", + "sampled_uint_1d"), + "integer 1D load did not access the uint 1D descriptor binding"); } void TestNewShaderRecompilerStorageImage1DDescriptorVariants() { - constexpr uint32_t MimgDim1D = 0; - constexpr uint32_t MimgDim1DArray = 4; - constexpr uint32_t SpirvDim1D = 0; + constexpr uint32_t MimgDim1D = 0; + constexpr uint32_t MimgDim1DArray = 4; + constexpr uint32_t SpirvDim1D = 0; - const uint32_t shader[] = { - EncodeMimg0(0x08, 0xf, false, MimgDim1D), - EncodeMimg1(20, 4, 0, 4), // image_store 1D - EncodeMimg0(0x08, 0xf, false, MimgDim1DArray), - EncodeMimg1(24, 5, 0, 8), // image_store 1D array - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf, false, MimgDim1D), + EncodeMimg1(20, 4, 0, 4), // image_store 1D + EncodeMimg0(0x08, 0xf, false, MimgDim1DArray), + EncodeMimg1(24, 5, 0, 8), // image_store 1D array + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - SetImageTestType(&user_data, 4, Prospero::ImageType::kColor1D); - SetImageTestType(&user_data, 5, Prospero::ImageType::kColor1DArray); + auto user_data = ImageTestUserData(); + SetImageTestType(&user_data, 4, Prospero::ImageType::kColor1D); + SetImageTestType(&user_data, 5, Prospero::ImageType::kColor1DArray); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "image_dim=1d"), - "1D storage dimension did not survive into IR metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_dim=1d_array"), - "1D-array storage dimension did not survive into IR metadata"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 0, 2), - "SPIR-V binary does not contain storage 1D image type"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 1, 2), - "SPIR-V binary does not contain storage 1D-array image type"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "image_dim=1d"), + "1D storage dimension did not survive into IR metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_dim=1d_array"), + "1D-array storage dimension did not survive into IR metadata"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 0, 2), + "SPIR-V binary does not contain storage 1D image type"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim1D, 1, 2), + "SPIR-V binary does not contain storage 1D-array image type"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_1d "), - "1D store did not access the 1D storage descriptor binding"); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_1d_array"), - "1D-array store did not access the 1D-array storage descriptor binding"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_1d "), + "1D store did not access the 1D storage descriptor binding"); + Check( + SpirvSourceHasInstructionUsing(source, "OpAccessChain", + "storage_1d_array"), + "1D-array store did not access the 1D-array storage descriptor binding"); } void TestNewShaderRecompilerNullImageUsesCanonical2DView() { - const uint32_t shader[] = { - EncodeMimg0(0x20, 0xf, false, 0), - EncodeMimg1(0, 0, 1, 0), // 1D instruction with a null image descriptor - 0xbf810000u, - }; - std::array user_data {}; + const uint32_t shader[] = { + EncodeMimg0(0x20, 0xf, false, 0), + EncodeMimg1(0, 0, 1, 0), // 1D instruction with a null image descriptor + 0xbf810000u, + }; + std::array user_data{}; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "sampled_2d "), - "null image did not use the canonical 2D sampled descriptor binding"); - Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "sampled_1d "), - "null image retained the decoded 1D descriptor binding"); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "sampled_2d "), + "null image did not use the canonical 2D sampled descriptor binding"); + Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "sampled_1d "), + "null image retained the decoded 1D descriptor binding"); } void TestNewShaderRecompilerRejectsOneDimensionalGather() { - constexpr std::array cases { - std::pair {0u, Prospero::ImageType::kColor1D}, - std::pair {4u, Prospero::ImageType::kColor1DArray}, - }; - for (const auto& [dimension, type]: cases) { - const uint32_t shader[] = { - EncodeMimg0(0x47, 0x1, false, dimension), - EncodeMimg1(0, 0, 1, 0), - 0xbf810000u, - }; - auto user_data = ImageTestUserData(type); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.user_data = user_data.data(); + constexpr std::array cases{ + std::pair{0u, Prospero::ImageType::kColor1D}, + std::pair{4u, Prospero::ImageType::kColor1DArray}, + }; + for (const auto &[dimension, type] : cases) { + const uint32_t shader[] = { + EncodeMimg0(0x47, 0x1, false, dimension), + EncodeMimg1(0, 0, 1, 0), + 0xbf810000u, + }; + auto user_data = ImageTestUserData(type); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && - Common::ContainsStr(error, "1D image gather is not supported by SPIR-V"), - "1D image gather escaped as invalid SPIR-V"); - } + ShaderRecompiler::CompileResult result; + std::string error; + Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && + Common::ContainsStr(error, + "1D image gather is not supported by SPIR-V"), + "1D image gather escaped as invalid SPIR-V"); + } } void TestNewShaderRecompilerImageGatherVariants() { - const uint32_t shader[] = { - EncodeMimg0(0x47, 0x1), - EncodeMimg1(60, 0, 1, 4), // image_gather4_lz - EncodeMimg0(0x48, 0x2), - EncodeMimg1(64, 0, 1, 8), // image_gather4_c - EncodeMimg0(0x4f, 0x1), - EncodeMimg1(68, 0, 1, 12), // image_gather4_c_lz - EncodeMimg0(0x57, 0x4), - EncodeMimg1(72, 0, 1, 16), // image_gather4_lz_o - EncodeMimg0(0x58, 0x1), - EncodeMimg1(76, 0, 1, 20), // image_gather4_c_o - EncodeMimg0(0x5f, 0x1), - EncodeMimg1(80, 0, 1, 24), // image_gather4_c_lz_o - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x47, 0x1), + EncodeMimg1(60, 0, 1, 4), // image_gather4_lz + EncodeMimg0(0x48, 0x2), + EncodeMimg1(64, 0, 1, 8), // image_gather4_c + EncodeMimg0(0x4f, 0x1), + EncodeMimg1(68, 0, 1, 12), // image_gather4_c_lz + EncodeMimg0(0x57, 0x4), + EncodeMimg1(72, 0, 1, 16), // image_gather4_lz_o + EncodeMimg0(0x58, 0x1), + EncodeMimg1(76, 0, 1, 20), // image_gather4_c_o + EncodeMimg0(0x5f, 0x1), + EncodeMimg1(80, 0, 1, 24), // image_gather4_c_lz_o + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz"), - "new decoder did not decode IMAGE_GATHER4_LZ"); - Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz_o"), - "new decoder did not decode IMAGE_GATHER4_LZ_O"); - Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c"), - "new decoder did not decode IMAGE_GATHER4_C"); - Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_lz"), - "new decoder did not decode IMAGE_GATHER4_C_LZ"); - Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_o"), - "new decoder did not decode IMAGE_GATHER4_C_O"); - Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_lz_o"), - "new decoder did not decode IMAGE_GATHER4_C_LZ_O"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0x4"), - "IMAGE_GATHER4_LZ_O did not preserve gather component dmask"); - Check(Common::ContainsStr(result.decoded_dump, - "sample_flags=offset|level_zero addr_components=3"), - "IMAGE_GATHER4_LZ_O did not expose shared offset sample metadata"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare addr_components=3"), - "IMAGE_GATHER4_C did not expose compare sample metadata"); - Check(Common::ContainsStr(result.decoded_dump, - "sample_flags=compare|level_zero addr_components=3"), - "IMAGE_GATHER4_C_LZ did not expose compare+level-zero sample metadata"); - Check(Common::ContainsStr(result.decoded_dump, "sample_flags=compare|offset addr_components=4"), - "IMAGE_GATHER4_C_O did not expose compare+offset sample metadata"); - Check(Common::ContainsStr(result.decoded_dump, - "sample_flags=compare|offset|level_zero addr_components=4"), - "IMAGE_GATHER4_C_LZ_O did not expose compare+offset+level-zero sample metadata"); - Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v60"), - "IMAGE_GATHER4_LZ did not lower to shared IR ImageGather4"); - Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v64"), - "IMAGE_GATHER4_C did not lower to shared IR ImageGather4"); - Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v68"), - "IMAGE_GATHER4_C_LZ did not lower to shared IR ImageGather4"); - Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v72"), - "IMAGE_GATHER4_LZ_O did not lower to shared IR ImageGather4"); - Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v76"), - "IMAGE_GATHER4_C_O did not lower to shared IR ImageGather4"); - Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v80"), - "IMAGE_GATHER4_C_LZ_O did not lower to shared IR ImageGather4"); - Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), - "IMAGE_GATHER4 did not preserve four-component result metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x30"), - "IMAGE_GATHER4_LZ_O flags did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x8"), - "IMAGE_GATHER4_C flags did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x28"), - "IMAGE_GATHER4_C_LZ flags did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x18"), - "IMAGE_GATHER4_C_O flags did not survive into IR memory metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_flags=0x38"), - "IMAGE_GATHER4_C_LZ_O flags did not survive into IR memory metadata"); - Check(SpirvContainsCapability(result.spirv, 25), - "SPIR-V binary does not request ImageGatherExtended"); - Check(SpirvContainsOpcode(result.spirv, 96), "SPIR-V binary does not contain OpImageGather"); - Check(SpirvContainsOpcode(result.spirv, 97), - "SPIR-V binary does not contain OpImageDrefGather"); - Check(SpirvContainsOpcode(result.spirv, 202), - "SPIR-V binary does not contain packed gather offset extraction"); - Check(SpirvContainsOpcode(result.spirv, 81), - "SPIR-V binary does not contain gather result extraction"); - Check(SpirvContainsOpcode(result.spirv, 124), - "SPIR-V binary does not contain gather result bitcast"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz"), + "new decoder did not decode IMAGE_GATHER4_LZ"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz_o"), + "new decoder did not decode IMAGE_GATHER4_LZ_O"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c"), + "new decoder did not decode IMAGE_GATHER4_C"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_lz"), + "new decoder did not decode IMAGE_GATHER4_C_LZ"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_o"), + "new decoder did not decode IMAGE_GATHER4_C_O"); + Check(Common::ContainsStr(result.decoded_dump, "image_gather4_c_lz_o"), + "new decoder did not decode IMAGE_GATHER4_C_LZ_O"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x4"), + "IMAGE_GATHER4_LZ_O did not preserve gather component dmask"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=offset|level_zero addr_components=3"), + "IMAGE_GATHER4_LZ_O did not expose shared offset sample metadata"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare addr_components=3"), + "IMAGE_GATHER4_C did not expose compare sample metadata"); + Check( + Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|level_zero addr_components=3"), + "IMAGE_GATHER4_C_LZ did not expose compare+level-zero sample metadata"); + Check(Common::ContainsStr(result.decoded_dump, + "sample_flags=compare|offset addr_components=4"), + "IMAGE_GATHER4_C_O did not expose compare+offset sample metadata"); + Check(Common::ContainsStr( + result.decoded_dump, + "sample_flags=compare|offset|level_zero addr_components=4"), + "IMAGE_GATHER4_C_LZ_O did not expose compare+offset+level-zero sample " + "metadata"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v60"), + "IMAGE_GATHER4_LZ did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v64"), + "IMAGE_GATHER4_C did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v68"), + "IMAGE_GATHER4_C_LZ did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v72"), + "IMAGE_GATHER4_LZ_O did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v76"), + "IMAGE_GATHER4_C_O did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "ImageGather4 v80"), + "IMAGE_GATHER4_C_LZ_O did not lower to shared IR ImageGather4"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), + "IMAGE_GATHER4 did not preserve four-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x30"), + "IMAGE_GATHER4_LZ_O flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x8"), + "IMAGE_GATHER4_C flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x28"), + "IMAGE_GATHER4_C_LZ flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x18"), + "IMAGE_GATHER4_C_O flags did not survive into IR memory metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_flags=0x38"), + "IMAGE_GATHER4_C_LZ_O flags did not survive into IR memory metadata"); + Check(SpirvContainsCapability(result.spirv, 25), + "SPIR-V binary does not request ImageGatherExtended"); + Check(SpirvContainsOpcode(result.spirv, 96), + "SPIR-V binary does not contain OpImageGather"); + Check(SpirvContainsOpcode(result.spirv, 97), + "SPIR-V binary does not contain OpImageDrefGather"); + Check(SpirvContainsOpcode(result.spirv, 202), + "SPIR-V binary does not contain packed gather offset extraction"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain gather result extraction"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain gather result bitcast"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageLoadVariants() { - const uint32_t shader[] = { - EncodeMimg0(0x00, 0x3), - EncodeMimg1(8, 0, 0, 1), // image_load dmask xy - EncodeMimg0(0x01, 0xf), - EncodeMimg1(12, 1, 0, 4), // image_load_mip dmask xyzw - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x00, 0x3), + EncodeMimg1(8, 0, 0, 1), // image_load dmask xy + EncodeMimg0(0x01, 0xf), + EncodeMimg1(12, 1, 0, 4), // image_load_mip dmask xyzw + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_load"), - "new decoder did not decode MIMG image load"); - Check(Common::ContainsStr(result.decoded_dump, "image_load_mip"), - "new decoder did not decode MIMG image load mip"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), - "image_load decode did not preserve partial dmask"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0xf"), - "image_load_mip decode did not preserve full dmask"); - Check(Common::ContainsStr(result.ir_dump, "ImageLoad v8"), - "image_load did not lower through shared image-load IR"); - Check(Common::ContainsStr(result.ir_dump, "ImageLoad v12"), - "image_load_mip did not lower through shared image-load IR"); - Check(Common::ContainsStr(result.ir_dump, "data_dwords=2"), - "image_load dmask xy did not preserve two-component result metadata"); - Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), - "image_load_mip dmask xyzw did not preserve four-component result metadata"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=1"), - "image_load_mip did not preserve mip address component metadata"); - Check(SpirvContainsOpcode(result.spirv, 95), "SPIR-V binary does not contain OpImageFetch"); - Check(SpirvContainsOpcode(result.spirv, 81), - "SPIR-V binary does not contain OpCompositeExtract for image-load dmask"); - Check(SpirvContainsOpcode(result.spirv, 124), - "SPIR-V binary does not contain OpBitcast for image-load result bits"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_load"), + "new decoder did not decode MIMG image load"); + Check(Common::ContainsStr(result.decoded_dump, "image_load_mip"), + "new decoder did not decode MIMG image load mip"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), + "image_load decode did not preserve partial dmask"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0xf"), + "image_load_mip decode did not preserve full dmask"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v8"), + "image_load did not lower through shared image-load IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageLoad v12"), + "image_load_mip did not lower through shared image-load IR"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=2"), + "image_load dmask xy did not preserve two-component result metadata"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), + "image_load_mip dmask xyzw did not preserve four-component result " + "metadata"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=1"), + "image_load_mip did not preserve mip address component metadata"); + Check(SpirvContainsOpcode(result.spirv, 95), + "SPIR-V binary does not contain OpImageFetch"); + Check( + SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain OpCompositeExtract for image-load dmask"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain OpBitcast for image-load result bits"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerImageLoad2DMsaa() { - const uint32_t shader[] = { - 0xf0000130u, // image_load v3, v[5:7], s[0:7] dmask:x dim:2d_msaa - 0x00000305u, - 0xbf810000u, - }; + const uint32_t shader[] = { + 0xf0000130u, // image_load v3, v[5:7], s[0:7] dmask:x dim:2d_msaa + 0x00000305u, + 0xbf810000u, + }; - auto user_data = ImageTestUserData(Prospero::ImageType::kColor2DMsaa); - user_data[3] |= 2u << 16u; - user_data[5] |= 2u << 4u; - user_data[6] |= 1u << 10u; + auto user_data = ImageTestUserData(Prospero::ImageType::kColor2DMsaa); + user_data[3] |= 2u << 16u; + user_data[5] |= 2u << 4u; + user_data[6] |= 1u << 10u; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.dump_ir = true; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_msaa") && - Common::ContainsStr(result.ir_dump, "image_dim=2d_msaa") && - Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=0"), - "RDNA2 2D-MSAA load did not preserve x, y, and fragment ID"); - Check(result.program.info.images.size() == 1 && - result.program.info.images[0].dimension == - ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa, - "2D-MSAA descriptor specialization lost the multisample dimension"); - Check(ShaderRecompiler::IR::FindBinding( - result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::Sampled2DMsaa) != nullptr, - "2D-MSAA image did not receive a multisampled descriptor binding"); - Check(SpirvContainsTypeImage(result.spirv, 1, 0, 1, 1), - "SPIR-V binary does not contain a multisampled 2D image type"); - CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "sampled_2d_msaa"), - "2D-MSAA load did not access its multisampled descriptor"); - Check(SpirvSourceHasInstructionUsing(source, "OpImageFetch", " Sample "), - "2D-MSAA load did not emit the fragment ID as a SPIR-V Sample operand"); - Check(!SpirvSourceHasInstructionUsing(source, "OpImageFetch", " Lod "), - "2D-MSAA load incorrectly emitted its fragment ID as a mip level"); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_msaa") && + Common::ContainsStr(result.ir_dump, "image_dim=2d_msaa") && + Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=0"), + "RDNA2 2D-MSAA load did not preserve x, y, and fragment ID"); + Check(result.program.info.images.size() == 1 && + result.program.info.images[0].dimension == + ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa, + "2D-MSAA descriptor specialization lost the multisample dimension"); + Check(ShaderRecompiler::IR::FindBinding( + result.program.bindings, + ShaderRecompiler::IR::DescriptorBindingKind::Sampled2DMsaa) != + nullptr, + "2D-MSAA image did not receive a multisampled descriptor binding"); + Check(SpirvContainsTypeImage(result.spirv, 1, 0, 1, 1), + "SPIR-V binary does not contain a multisampled 2D image type"); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", + "sampled_2d_msaa"), + "2D-MSAA load did not access its multisampled descriptor"); + Check(SpirvSourceHasInstructionUsing(source, "OpImageFetch", " Sample "), + "2D-MSAA load did not emit the fragment ID as a SPIR-V Sample operand"); + Check(!SpirvSourceHasInstructionUsing(source, "OpImageFetch", " Lod "), + "2D-MSAA load incorrectly emitted its fragment ID as a mip level"); } void TestNewShaderRecompilerImageStoreLowering() { - const uint32_t shader[] = { - EncodeMimg0(0x08, 0xf), - EncodeMimg1(20, 0, 0, 4), // image_store - EncodeMimg0(0x09, 0x3), - EncodeMimg1(24, 1, 0, 8), // image_store_mip - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf), + EncodeMimg1(20, 0, 0, 4), // image_store + EncodeMimg0(0x09, 0x3), + EncodeMimg1(24, 1, 0, 8), // image_store_mip + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_store"), - "new decoder did not decode MIMG image store"); - Check(Common::ContainsStr(result.decoded_dump, "image_store_mip"), - "new decoder did not decode MIMG image store mip"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0xf"), - "image store decode did not preserve full dmask"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), - "image store mip decode did not preserve partial dmask"); - Check(Common::ContainsStr(result.ir_dump, "ImageStore null, v20, v4"), - "image_store did not lower to shared image-store IR"); - Check(Common::ContainsStr(result.ir_dump, "ImageStore null, v24, v8"), - "image_store_mip did not lower to shared image-store IR"); - Check(Common::ContainsStr(result.ir_dump, "storage_image"), - "image store IR did not use storage-image resource metadata"); - Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), - "full-dmask image store did not preserve data component count"); - Check(Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=1"), - "mip image store did not preserve mip address component count"); - Check(SpirvContainsOpcode(result.spirv, 99), "SPIR-V binary does not contain OpImageWrite"); - Check(SpirvContainsOpcode(result.spirv, 80), - "SPIR-V binary does not contain image texel/coordinate composites"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_store"), + "new decoder did not decode MIMG image store"); + Check(Common::ContainsStr(result.decoded_dump, "image_store_mip"), + "new decoder did not decode MIMG image store mip"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0xf"), + "image store decode did not preserve full dmask"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), + "image store mip decode did not preserve partial dmask"); + Check(Common::ContainsStr(result.ir_dump, "ImageStore null, v20, v4"), + "image_store did not lower to shared image-store IR"); + Check(Common::ContainsStr(result.ir_dump, "ImageStore null, v24, v8"), + "image_store_mip did not lower to shared image-store IR"); + Check(Common::ContainsStr(result.ir_dump, "storage_image"), + "image store IR did not use storage-image resource metadata"); + Check(Common::ContainsStr(result.ir_dump, "data_dwords=4"), + "full-dmask image store did not preserve data component count"); + Check(Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=1"), + "mip image store did not preserve mip address component count"); + Check(SpirvContainsOpcode(result.spirv, 99), + "SPIR-V binary does not contain OpImageWrite"); + Check(SpirvContainsOpcode(result.spirv, 80), + "SPIR-V binary does not contain image texel/coordinate composites"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerStorageImage3DDescriptorVariant() { - constexpr uint32_t ImageTypeColor3D = 10; - constexpr uint32_t ImageFormatRgba16f = 71; - constexpr uint32_t SpirvDim3D = 2; + constexpr uint32_t ImageTypeColor3D = 10; + constexpr uint32_t ImageFormatRgba16f = 71; + constexpr uint32_t SpirvDim3D = 2; - constexpr uint32_t MimgDim3D = 2; + constexpr uint32_t MimgDim3D = 2; - const uint32_t shader[] = { - EncodeMimg0(0x08, 0xf, false, MimgDim3D), - EncodeMimg1(20, 5, 0, 4), // image_store 3D - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf, false, MimgDim3D), + EncodeMimg1(20, 5, 0, 4), // image_store 3D + 0xbf810000u, + }; - ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); - std::array user_data {}; - user_data[20] = 0x1000u; - user_data[21] = ImageFormatRgba16f << 20u; - user_data[22] = 255u | (255u << 14u); - user_data[23] = ImageTypeColor3D << 28u; + ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); + std::array user_data{}; + user_data[20] = 0x1000u; + user_data[21] = ImageFormatRgba16f << 20u; + user_data[22] = 255u | (255u << 14u); + user_data[23] = ImageTypeColor3D << 28u; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.compute_input_info = &input_info; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = &input_info; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), - "MIMG store did not decode the RDNA2 3D instruction dimension"); - const auto has_3d_store = std::any_of( - result.program.blocks.begin(), result.program.blocks.end(), [](const auto& block) { - return std::any_of(block.instructions.begin(), block.instructions.end(), - [](const auto& inst) { - return inst.op == ShaderRecompiler::IR::Opcode::ImageStore && - inst.memory.image_dimension == - ShaderRecompiler::Decoder::ImageDimension::Dim3D && - inst.memory.image_address_components == 3; - }); - }); - Check(has_3d_store, "3D storage image store did not preserve three address components"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim3D, 0, 2), - "SPIR-V binary does not contain storage 3D image type"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), + "MIMG store did not decode the RDNA2 3D instruction dimension"); + const auto has_3d_store = std::any_of( + result.program.blocks.begin(), result.program.blocks.end(), + [](const auto &block) { + return std::any_of( + block.instructions.begin(), block.instructions.end(), + [](const auto &inst) { + return inst.op == ShaderRecompiler::IR::Opcode::ImageStore && + inst.memory.image_dimension == + ShaderRecompiler::Decoder::ImageDimension::Dim3D && + inst.memory.image_address_components == 3; + }); + }); + Check(has_3d_store, + "3D storage image store did not preserve three address components"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim3D, 0, 2), + "SPIR-V binary does not contain storage 3D image type"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_3d"), - "storage image store did not access the 3D storage descriptor binding"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_3d"), + "storage image store did not access the 3D storage descriptor binding"); } void TestNewShaderRecompilerStorageImage2DDescriptorOverridesMimg3D() { - constexpr uint32_t ImageTypeColor2D = 9; - constexpr uint32_t ImageFormatRgba16f = 71; - constexpr uint32_t MimgDim3D = 2; - constexpr uint32_t SpirvDim2D = 1; + constexpr uint32_t ImageTypeColor2D = 9; + constexpr uint32_t ImageFormatRgba16f = 71; + constexpr uint32_t MimgDim3D = 2; + constexpr uint32_t SpirvDim2D = 1; - const uint32_t shader[] = { - EncodeMimg0(0x08, 0xf, false, MimgDim3D), - EncodeMimg1(20, 5, 0, 4), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x08, 0xf, false, MimgDim3D), + EncodeMimg1(20, 5, 0, 4), + 0xbf810000u, + }; - ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); - std::array user_data {}; - user_data[20] = 0x1000u; - user_data[21] = ImageFormatRgba16f << 20u; - user_data[22] = 255u | (255u << 14u); - user_data[23] = ImageTypeColor2D << 28u; + ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); + std::array user_data{}; + user_data[20] = 0x1000u; + user_data[21] = ImageFormatRgba16f << 20u; + user_data[22] = 255u | (255u << 14u); + user_data[23] = ImageTypeColor2D << 28u; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.compute_input_info = &input_info; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = &input_info; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), - "test MIMG store should decode as a 3D instruction"); - Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 0, 2), - "SPIR-V binary does not contain storage 2D image type"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), + "test MIMG store should decode as a 3D instruction"); + Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 0, 2), + "SPIR-V binary does not contain storage 2D image type"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_2d"), - "2D descriptor storage image store did not access the base storage binding"); - Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_2d_array"), - "2D descriptor storage image store unexpectedly used the array storage binding"); - Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_3d"), - "2D descriptor storage image store unexpectedly used the 3D storage binding"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_2d"), + "2D descriptor storage image store did not access the base storage " + "binding"); + Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", + "storage_2d_array"), + "2D descriptor storage image store unexpectedly used the array storage " + "binding"); + Check(!SpirvSourceHasInstructionUsing(source, "OpAccessChain", "storage_3d"), + "2D descriptor storage image store unexpectedly used the 3D storage " + "binding"); } void TestNewShaderRecompilerImageAtomicLowering() { - const uint32_t shader[] = { - EncodeMimg0(0x11, 0x1, true), - EncodeMimg1(52, 0, 0, 1), // image_atomic_add - EncodeMimg0(0x15, 0x1, true), - EncodeMimg1(53, 0, 0, 1), // image_atomic_umin - EncodeMimg0(0x17, 0x1, true), - EncodeMimg1(57, 0, 0, 1), // image_atomic_umax - EncodeMimg0(0x18, 0x1, true), - EncodeMimg1(54, 0, 0, 1), // image_atomic_and - EncodeMimg0(0x19, 0x1, true), - EncodeMimg1(55, 0, 0, 1), // image_atomic_or - EncodeMimg0(0x1a, 0x1, true), - EncodeMimg1(56, 0, 0, 1), // image_atomic_xor - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMimg0(0x11, 0x1, true), + EncodeMimg1(52, 0, 0, 1), // image_atomic_add + EncodeMimg0(0x15, 0x1, true), + EncodeMimg1(53, 0, 0, 1), // image_atomic_umin + EncodeMimg0(0x17, 0x1, true), + EncodeMimg1(57, 0, 0, 1), // image_atomic_umax + EncodeMimg0(0x18, 0x1, true), + EncodeMimg1(54, 0, 0, 1), // image_atomic_and + EncodeMimg0(0x19, 0x1, true), + EncodeMimg1(55, 0, 0, 1), // image_atomic_or + EncodeMimg0(0x1a, 0x1, true), + EncodeMimg1(56, 0, 0, 1), // image_atomic_xor + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "image_atomic_add"), - "new decoder did not decode MIMG image atomic add"); - Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umin"), - "new decoder did not decode MIMG image atomic umin"); - Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umax"), - "new decoder did not decode MIMG image atomic umax"); - Check(Common::ContainsStr(result.decoded_dump, "image_atomic_and"), - "new decoder did not decode MIMG image atomic and"); - Check(Common::ContainsStr(result.decoded_dump, "image_atomic_or"), - "new decoder did not decode MIMG image atomic or"); - Check(Common::ContainsStr(result.decoded_dump, "image_atomic_xor"), - "new decoder did not decode MIMG image atomic xor"); - Check(Common::ContainsStr(result.decoded_dump, "dmask=0x1"), - "image atomic decode did not preserve dmask metadata"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v52, v52, v1"), - "image_atomic_add did not lower through shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v53, v53, v1"), - "image_atomic_umin did not lower through shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v57, v57, v1"), - "image_atomic_umax did not lower through shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v54, v54, v1"), - "image_atomic_and did not lower through shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v55, v55, v1"), - "image_atomic_or did not lower through shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v56, v56, v1"), - "image_atomic_xor did not lower through shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "storage_image_uint"), - "image atomic IR did not use uint storage-image metadata"); - Check(SpirvContainsOpcode(result.spirv, 60), - "SPIR-V binary does not contain OpImageTexelPointer"); - Check(SpirvContainsOpcode(result.spirv, 234), "SPIR-V binary does not contain OpAtomicIAdd"); - Check(SpirvContainsOpcode(result.spirv, 237), "SPIR-V binary does not contain OpAtomicUMin"); - Check(SpirvContainsOpcode(result.spirv, 239), "SPIR-V binary does not contain OpAtomicUMax"); - Check(SpirvContainsOpcode(result.spirv, 240), "SPIR-V binary does not contain OpAtomicAnd"); - Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr"); - Check(SpirvContainsOpcode(result.spirv, 242), "SPIR-V binary does not contain OpAtomicXor"); - Check(SpirvContainsOpcode(result.spirv, 225), - "image atomic SPIR-V binary does not contain OpMemoryBarrier"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_add"), + "new decoder did not decode MIMG image atomic add"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umin"), + "new decoder did not decode MIMG image atomic umin"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umax"), + "new decoder did not decode MIMG image atomic umax"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_and"), + "new decoder did not decode MIMG image atomic and"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_or"), + "new decoder did not decode MIMG image atomic or"); + Check(Common::ContainsStr(result.decoded_dump, "image_atomic_xor"), + "new decoder did not decode MIMG image atomic xor"); + Check(Common::ContainsStr(result.decoded_dump, "dmask=0x1"), + "image atomic decode did not preserve dmask metadata"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v52, v52, v1"), + "image_atomic_add did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v53, v53, v1"), + "image_atomic_umin did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v57, v57, v1"), + "image_atomic_umax did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v54, v54, v1"), + "image_atomic_and did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v55, v55, v1"), + "image_atomic_or did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v56, v56, v1"), + "image_atomic_xor did not lower through shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "storage_image_uint"), + "image atomic IR did not use uint storage-image metadata"); + Check(SpirvContainsOpcode(result.spirv, 60), + "SPIR-V binary does not contain OpImageTexelPointer"); + Check(SpirvContainsOpcode(result.spirv, 234), + "SPIR-V binary does not contain OpAtomicIAdd"); + Check(SpirvContainsOpcode(result.spirv, 237), + "SPIR-V binary does not contain OpAtomicUMin"); + Check(SpirvContainsOpcode(result.spirv, 239), + "SPIR-V binary does not contain OpAtomicUMax"); + Check(SpirvContainsOpcode(result.spirv, 240), + "SPIR-V binary does not contain OpAtomicAnd"); + Check(SpirvContainsOpcode(result.spirv, 241), + "SPIR-V binary does not contain OpAtomicOr"); + Check(SpirvContainsOpcode(result.spirv, 242), + "SPIR-V binary does not contain OpAtomicXor"); + Check(SpirvContainsOpcode(result.spirv, 225), + "image atomic SPIR-V binary does not contain OpMemoryBarrier"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerVintrpLowering() { - const uint32_t shader[] = { - EncodeVintrp(0, 10, 1, 2, 4), // v_interp_p1_f32 v10, v4, attr1.z - EncodeVintrp(1, 11, 1, 2, 4), // v_interp_p2_f32 v11, v4, attr1.z - EncodeVintrp(2, 12, 0, 3, 2), // v_interp_mov_f32 v12, p0, attr0.w - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeVintrp(0, 10, 1, 2, 4), // v_interp_p1_f32 v10, v4, attr1.z + EncodeVintrp(1, 11, 1, 2, 4), // v_interp_p2_f32 v11, v4, attr1.z + EncodeVintrp(2, 12, 0, 3, 2), // v_interp_mov_f32 v12, p0, attr0.w + 0xbf810000u, + }; - ShaderPixelInputInfo ps_info {}; - ps_info.input_num = 2; - SetIdentityInterpolatorSettings(&ps_info); + ShaderPixelInputInfo ps_info{}; + ps_info.input_num = 2; + SetIdentityInterpolatorSettings(&ps_info); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.pixel_input_info = &ps_info; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "v_interp_p1_f32"), - "new decoder did not decode VINTRP P1"); - Check(Common::ContainsStr(result.decoded_dump, "v_interp_p2_f32"), - "new decoder did not decode VINTRP P2"); - Check(Common::ContainsStr(result.decoded_dump, "v_interp_mov_f32"), - "new decoder did not decode VINTRP MOV"); - Check(Common::ContainsStr(result.ir_dump, "ControlNop"), - "VINTRP P1 did not lower to an explicit no-op marker"); - Check(Common::ContainsStr(result.ir_dump, "LoadInputF32 v11"), - "VINTRP P2 did not lower to input-load IR"); - Check(Common::ContainsStr(result.ir_dump, "input_attr=1 input_chan=2"), - "VINTRP P2 did not preserve attr/channel metadata"); - Check(Common::ContainsStr(result.ir_dump, "LoadInputF32 v12"), - "VINTRP MOV did not lower to input-load IR"); - Check(Common::ContainsStr(result.ir_dump, "input_attr=0 input_chan=3"), - "VINTRP MOV did not preserve attr/channel metadata"); - Check(ProgramInputCount(result.program, ShaderRecompiler::IR::StageInputKind::Parameter) == 2, - "VINTRP pixel shader did not reflect parameter inputs"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 81), - "SPIR-V binary does not contain input component extraction"); - Check(SpirvContainsOpcode(result.spirv, 124), "SPIR-V binary does not contain input bitcast"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_interp_p1_f32"), + "new decoder did not decode VINTRP P1"); + Check(Common::ContainsStr(result.decoded_dump, "v_interp_p2_f32"), + "new decoder did not decode VINTRP P2"); + Check(Common::ContainsStr(result.decoded_dump, "v_interp_mov_f32"), + "new decoder did not decode VINTRP MOV"); + Check(Common::ContainsStr(result.ir_dump, "ControlNop"), + "VINTRP P1 did not lower to an explicit no-op marker"); + Check(Common::ContainsStr(result.ir_dump, "LoadInputF32 v11"), + "VINTRP P2 did not lower to input-load IR"); + Check(Common::ContainsStr(result.ir_dump, "input_attr=1 input_chan=2"), + "VINTRP P2 did not preserve attr/channel metadata"); + Check(Common::ContainsStr(result.ir_dump, "LoadInputF32 v12"), + "VINTRP MOV did not lower to input-load IR"); + Check(Common::ContainsStr(result.ir_dump, "input_attr=0 input_chan=3"), + "VINTRP MOV did not preserve attr/channel metadata"); + Check(ProgramInputCount(result.program, + ShaderRecompiler::IR::StageInputKind::Parameter) == 2, + "VINTRP pixel shader did not reflect parameter inputs"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 81), + "SPIR-V binary does not contain input component extraction"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain input bitcast"); + CheckSpirvBinaryValidates(result.spirv); - const uint32_t remapped_shader[] = { - EncodeVintrp(1, 11, 2, 0, 4), // v_interp_p2_f32 v11, v4, attr2.x - 0xbf810000u, - }; - ShaderPixelInputInfo remapped_ps_info {}; - remapped_ps_info.input_num = 3; - SetIdentityInterpolatorSettings(&remapped_ps_info); - remapped_ps_info.interpolator_settings[2] = 3; + const uint32_t remapped_shader[] = { + EncodeVintrp(1, 11, 2, 0, 4), // v_interp_p2_f32 v11, v4, attr2.x + 0xbf810000u, + }; + ShaderPixelInputInfo remapped_ps_info{}; + remapped_ps_info.input_num = 3; + SetIdentityInterpolatorSettings(&remapped_ps_info); + remapped_ps_info.interpolator_settings[2] = 3; - options.pixel_input_info = &remapped_ps_info; + options.pixel_input_info = &remapped_ps_info; - ShaderRecompiler::CompileResult remapped_result; - Check(ShaderRecompiler::TryRecompile(remapped_shader, options, remapped_result, &error), - error.c_str()); - Check(Common::ContainsStr(remapped_result.ir_dump, "input_attr=2 input_chan=0"), - "remapped VINTRP did not preserve raw pixel attribute metadata"); - Check(SpirvHasDecorationValue(remapped_result.spirv, 30u, 3u), - "remapped VINTRP did not use SPI_PS_INPUT_CNTL input offset as Location"); - Check(!SpirvHasDecorationValue(remapped_result.spirv, 30u, 2u), - "remapped VINTRP still emitted the raw attribute as Location"); - CheckSpirvBinaryValidates(remapped_result.spirv); + ShaderRecompiler::CompileResult remapped_result; + Check(ShaderRecompiler::TryRecompile(remapped_shader, options, + remapped_result, &error), + error.c_str()); + Check( + Common::ContainsStr(remapped_result.ir_dump, "input_attr=2 input_chan=0"), + "remapped VINTRP did not preserve raw pixel attribute metadata"); + Check( + SpirvHasDecorationValue(remapped_result.spirv, 30u, 3u), + "remapped VINTRP did not use SPI_PS_INPUT_CNTL input offset as Location"); + Check(!SpirvHasDecorationValue(remapped_result.spirv, 30u, 2u), + "remapped VINTRP still emitted the raw attribute as Location"); + CheckSpirvBinaryValidates(remapped_result.spirv); - const uint32_t duplicate_location_shader[] = { - EncodeVintrp(0, 6, 1, 0, 4), // v_interp_p1_f32 v6, v4, attr1.x - EncodeVintrp(0, 10, 0, 0, 4), // v_interp_p1_f32 v10, v4, attr0.x - EncodeVintrp(1, 6, 1, 0, 4), // v_interp_p2_f32 v6, v4, attr1.x - EncodeVintrp(1, 10, 0, 0, 4), // v_interp_p2_f32 v10, v4, attr0.x - 0xbf810000u, - }; - ShaderPixelInputInfo duplicate_ps_info {}; - duplicate_ps_info.input_num = 2; + const uint32_t duplicate_location_shader[] = { + EncodeVintrp(0, 6, 1, 0, 4), // v_interp_p1_f32 v6, v4, attr1.x + EncodeVintrp(0, 10, 0, 0, 4), // v_interp_p1_f32 v10, v4, attr0.x + EncodeVintrp(1, 6, 1, 0, 4), // v_interp_p2_f32 v6, v4, attr1.x + EncodeVintrp(1, 10, 0, 0, 4), // v_interp_p2_f32 v10, v4, attr0.x + 0xbf810000u, + }; + ShaderPixelInputInfo duplicate_ps_info{}; + duplicate_ps_info.input_num = 2; - options.pixel_input_info = &duplicate_ps_info; + options.pixel_input_info = &duplicate_ps_info; - ShaderRecompiler::CompileResult duplicate_result; - Check(ShaderRecompiler::TryRecompile(duplicate_location_shader, options, duplicate_result, - &error), - error.c_str()); - Check(ProgramInputCount(duplicate_result.program, - ShaderRecompiler::IR::StageInputKind::Parameter) == 2, - "duplicate-location VINTRP shader did not reflect both raw parameter attrs"); - Check(SpirvDecorationValueCount(duplicate_result.spirv, 30u, 0u) == 1, - "duplicate-location VINTRP emitted more than one Location 0 input"); - Check(SpirvDecorationValueCount(duplicate_result.spirv, 30u, 1u) == 1, - "duplicate-location VINTRP did not fall back attr1 to Location 1"); - CheckSpirvBinaryValidates(duplicate_result.spirv); + ShaderRecompiler::CompileResult duplicate_result; + Check(ShaderRecompiler::TryRecompile(duplicate_location_shader, options, + duplicate_result, &error), + error.c_str()); + Check(ProgramInputCount(duplicate_result.program, + ShaderRecompiler::IR::StageInputKind::Parameter) == 2, + "duplicate-location VINTRP shader did not reflect both raw parameter " + "attrs"); + Check(SpirvDecorationValueCount(duplicate_result.spirv, 30u, 0u) == 1, + "duplicate-location VINTRP emitted more than one Location 0 input"); + Check(SpirvDecorationValueCount(duplicate_result.spirv, 30u, 1u) == 1, + "duplicate-location VINTRP did not fall back attr1 to Location 1"); + CheckSpirvBinaryValidates(duplicate_result.spirv); - const uint32_t flat_shader[] = { - EncodeVintrp(2, 12, 0, 0, 2), // v_interp_mov_f32 v12, p0, attr0.x - 0xbf810000u, - }; - ShaderPixelInputInfo flat_ps_info {}; - flat_ps_info.input_num = 1; - SetIdentityInterpolatorSettings(&flat_ps_info); - flat_ps_info.interpolator_settings[0] = 0x00000400u; + const uint32_t flat_shader[] = { + EncodeVintrp(2, 12, 0, 0, 2), // v_interp_mov_f32 v12, p0, attr0.x + 0xbf810000u, + }; + ShaderPixelInputInfo flat_ps_info{}; + flat_ps_info.input_num = 1; + SetIdentityInterpolatorSettings(&flat_ps_info); + flat_ps_info.interpolator_settings[0] = 0x00000400u; - options.pixel_input_info = &flat_ps_info; + options.pixel_input_info = &flat_ps_info; - ShaderRecompiler::CompileResult flat_result; - Check(ShaderRecompiler::TryRecompile(flat_shader, options, flat_result, &error), error.c_str()); - Check(SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 14u), - "flat VINTRP input did not emit a Flat decoration"); - Check(!SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 13u), - "flat VINTRP input should not also emit NoPerspective"); - Check(SpirvHasDecorationValue(flat_result.spirv, 30u, 0u), - "flat VINTRP input did not preserve Location 0"); - CheckSpirvBinaryValidates(flat_result.spirv); + ShaderRecompiler::CompileResult flat_result; + Check( + ShaderRecompiler::TryRecompile(flat_shader, options, flat_result, &error), + error.c_str()); + Check(SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 14u), + "flat VINTRP input did not emit a Flat decoration"); + Check(!SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 13u), + "flat VINTRP input should not also emit NoPerspective"); + Check(SpirvHasDecorationValue(flat_result.spirv, 30u, 0u), + "flat VINTRP input did not preserve Location 0"); + CheckSpirvBinaryValidates(flat_result.spirv); - ShaderPixelInputInfo no_perspective_ps_info {}; - no_perspective_ps_info.input_num = 1; - no_perspective_ps_info.ps_no_perspective = true; - SetIdentityInterpolatorSettings(&no_perspective_ps_info); + ShaderPixelInputInfo no_perspective_ps_info{}; + no_perspective_ps_info.input_num = 1; + no_perspective_ps_info.ps_no_perspective = true; + SetIdentityInterpolatorSettings(&no_perspective_ps_info); - options.pixel_input_info = &no_perspective_ps_info; + options.pixel_input_info = &no_perspective_ps_info; - ShaderRecompiler::CompileResult no_perspective_result; - Check(ShaderRecompiler::TryRecompile(flat_shader, options, no_perspective_result, &error), - error.c_str()); - Check(SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, 0u, 13u), - "no-perspective VINTRP input did not emit a NoPerspective decoration"); - Check(!SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, 0u, 14u), - "non-flat no-perspective VINTRP input should not emit Flat"); - CheckSpirvBinaryValidates(no_perspective_result.spirv); + ShaderRecompiler::CompileResult no_perspective_result; + Check(ShaderRecompiler::TryRecompile(flat_shader, options, + no_perspective_result, &error), + error.c_str()); + Check(SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, + 0u, 13u), + "no-perspective VINTRP input did not emit a NoPerspective decoration"); + Check(!SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, + 0u, 14u), + "non-flat no-perspective VINTRP input should not emit Flat"); + CheckSpirvBinaryValidates(no_perspective_result.spirv); } void TestPsInputCountRegisterDecode() { - HW::Context context; - // NUM_INTERP is 3 while bit 14 is an independent control flag that must be preserved. - context.SetPsInControl(0x00004003u); - const auto ps_in_control = context.GetShaderRegisters().ps_in_control; - Check(ps_in_control == 0x00004003u, "SPI_PS_IN_CONTROL flags were not preserved"); - Check((ps_in_control & 0x3fu) == 3, "SPI_PS_IN_CONTROL NUM_INTERP decoding failed"); + HW::Context context; + // NUM_INTERP is 3 while bit 14 is an independent control flag that must be + // preserved. + context.SetPsInControl(0x00004003u); + const auto ps_in_control = context.GetShaderRegisters().ps_in_control; + Check(ps_in_control == 0x00004003u, + "SPI_PS_IN_CONTROL flags were not preserved"); + Check((ps_in_control & 0x3fu) == 3, + "SPI_PS_IN_CONTROL NUM_INTERP decoding failed"); } void TestGraphicsCreateInterpolantMapping() { - ShaderRegister regs[32] = {}; + ShaderRegister regs[32] = {}; - Check(Gen5::GraphicsCreateInterpolantMapping(regs, nullptr, nullptr) == 0, - "null pixel shader interpolant mapping failed"); - for (uint32_t i = 0; i < 32u; i++) { - Check(regs[i].offset == Pm4::CX_PS_SHADER_USAGE_BASE + i, - "identity interpolant register offset was unexpected"); - Check(regs[i].value == i, "identity interpolant register value was unexpected"); - } + Check(Gen5::GraphicsCreateInterpolantMapping(regs, nullptr, nullptr) == 0, + "null pixel shader interpolant mapping failed"); + for (uint32_t i = 0; i < 32u; i++) { + Check(regs[i].offset == Pm4::CX_PS_SHADER_USAGE_BASE + i, + "identity interpolant register offset was unexpected"); + Check(regs[i].value == i, + "identity interpolant register value was unexpected"); + } - ShaderSemantic gs_semantics[3] = {}; - gs_semantics[0].semantic = 7; - gs_semantics[0].hardware_mapping = 5; - gs_semantics[1].semantic = 9; - gs_semantics[1].hardware_mapping = 12; - gs_semantics[2].semantic = 10; - gs_semantics[2].hardware_mapping = 4; - gs_semantics[2].is_f16 = 1; + ShaderSemantic gs_semantics[3] = {}; + gs_semantics[0].semantic = 7; + gs_semantics[0].hardware_mapping = 5; + gs_semantics[1].semantic = 9; + gs_semantics[1].hardware_mapping = 12; + gs_semantics[2].semantic = 10; + gs_semantics[2].hardware_mapping = 4; + gs_semantics[2].is_f16 = 1; - Shader gs {}; - gs.output_semantics = gs_semantics; - gs.num_output_semantics = static_cast(std::size(gs_semantics)); + Shader gs{}; + gs.output_semantics = gs_semantics; + gs.num_output_semantics = static_cast(std::size(gs_semantics)); - ShaderSemantic ps_semantics[4] = {}; - ps_semantics[0].semantic = 7; - ps_semantics[1].semantic = 8; - ps_semantics[1].default_value = 2; - ps_semantics[2].semantic = 9; - ps_semantics[2].is_flat_shaded = 1; - ps_semantics[2].is_custom = 1; - ps_semantics[2].default_value = 1; - ps_semantics[3].semantic = 10; - ps_semantics[3].is_f16 = 1; - ps_semantics[3].default_value = 3; - ps_semantics[3].default_value_hi = 2; + ShaderSemantic ps_semantics[4] = {}; + ps_semantics[0].semantic = 7; + ps_semantics[1].semantic = 8; + ps_semantics[1].default_value = 2; + ps_semantics[2].semantic = 9; + ps_semantics[2].is_flat_shaded = 1; + ps_semantics[2].is_custom = 1; + ps_semantics[2].default_value = 1; + ps_semantics[3].semantic = 10; + ps_semantics[3].is_f16 = 1; + ps_semantics[3].default_value = 3; + ps_semantics[3].default_value_hi = 2; - Shader ps {}; - ps.input_semantics = ps_semantics; - ps.num_input_semantics = static_cast(std::size(ps_semantics)); + Shader ps{}; + ps.input_semantics = ps_semantics; + ps.num_input_semantics = static_cast(std::size(ps_semantics)); - Check(Gen5::GraphicsCreateInterpolantMapping(regs, &gs, &ps) == 0, - "shader interpolant mapping failed"); - Check(regs[0].value == 0x00000005u, - "matched interpolant mapping did not use GS hardware mapping"); - Check(regs[1].value == 0x00000220u, "missing interpolant mapping did not use PS default value"); - Check(regs[2].value == 0x0000052cu, "flat/custom interpolant mapping bits were unexpected"); - Check(regs[3].value == 0x01580304u, "f16 interpolant mapping bits were unexpected"); - Check(regs[4].offset == Pm4::CX_PS_SHADER_USAGE_BASE + 4u && regs[4].value == 4u, - "interpolant identity tail was not filled"); + Check(Gen5::GraphicsCreateInterpolantMapping(regs, &gs, &ps) == 0, + "shader interpolant mapping failed"); + Check(regs[0].value == 0x00000005u, + "matched interpolant mapping did not use GS hardware mapping"); + Check(regs[1].value == 0x00000220u, + "missing interpolant mapping did not use PS default value"); + Check(regs[2].value == 0x0000052cu, + "flat/custom interpolant mapping bits were unexpected"); + Check(regs[3].value == 0x01580304u, + "f16 interpolant mapping bits were unexpected"); + Check(regs[4].offset == Pm4::CX_PS_SHADER_USAGE_BASE + 4u && + regs[4].value == 4u, + "interpolant identity tail was not filled"); } void TestNewShaderRecompilerWideMemoryLowering() { - const uint32_t shader[] = { - EncodeSmem0(0x02, 4, 4), - 0u, // s_load_dwordx4 s[4:7] - EncodeSmem0(0x0a, 8, 4), - 0u, // s_buffer_load_dwordx4 s[8:11] - EncodeSmem0(0x09, 106, 4), - 0u, // s_buffer_load_dwordx2 vcc_lo - EncodeMubuf0(0x0d, 16), - EncodeMubuf1(20, 0, 1), // buffer_load_dwordx2 v[20:21] - EncodeMubuf0(0x0f, 32), - EncodeMubuf1(24, 0, 1), // buffer_load_dwordx3 v[24:26] - EncodeMubuf0(0x0e, 48), - EncodeMubuf1(28, 0, 1), // buffer_load_dwordx4 v[28:31] - EncodeMubuf0(0x1d, 64), - EncodeMubuf1(32, 0, 1), // buffer_store_dwordx2 v[32:33] - EncodeMubuf0(0x1f, 80), - EncodeMubuf1(36, 0, 1), // buffer_store_dwordx3 v[36:38] - EncodeMubuf0(0x1e, 96), - EncodeMubuf1(40, 0, 1), // buffer_store_dwordx4 v[40:43] - EncodeMubuf0(0x08, 2), - EncodeMubuf1(44, 0, 1), // buffer_load_ubyte v44 - EncodeMubuf0(0x0a, 4), - EncodeMubuf1(45, 0, 1), // buffer_load_ushort v45 - EncodeFlat0(0x0c, 0, 4), - EncodeFlat1(50, 0x7d, 0, 1), // flat_load_dword - EncodeFlat0(0x0d, 1, 8), - EncodeFlat1(52, 0x7d, 0, 1), // scratch_load_dwordx2 - EncodeFlat0(0x0e, 2, 12), - EncodeFlat1(56, 0x7d, 0, 1), // global_load_dwordx4 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSmem0(0x02, 4, 4), + 0u, // s_load_dwordx4 s[4:7] + EncodeSmem0(0x0a, 8, 4), + 0u, // s_buffer_load_dwordx4 s[8:11] + EncodeSmem0(0x09, 106, 4), + 0u, // s_buffer_load_dwordx2 vcc_lo + EncodeMubuf0(0x0d, 16), + EncodeMubuf1(20, 0, 1), // buffer_load_dwordx2 v[20:21] + EncodeMubuf0(0x0f, 32), + EncodeMubuf1(24, 0, 1), // buffer_load_dwordx3 v[24:26] + EncodeMubuf0(0x0e, 48), + EncodeMubuf1(28, 0, 1), // buffer_load_dwordx4 v[28:31] + EncodeMubuf0(0x1d, 64), + EncodeMubuf1(32, 0, 1), // buffer_store_dwordx2 v[32:33] + EncodeMubuf0(0x1f, 80), + EncodeMubuf1(36, 0, 1), // buffer_store_dwordx3 v[36:38] + EncodeMubuf0(0x1e, 96), + EncodeMubuf1(40, 0, 1), // buffer_store_dwordx4 v[40:43] + EncodeMubuf0(0x08, 2), + EncodeMubuf1(44, 0, 1), // buffer_load_ubyte v44 + EncodeMubuf0(0x0a, 4), + EncodeMubuf1(45, 0, 1), // buffer_load_ushort v45 + EncodeFlat0(0x0c, 0, 4), + EncodeFlat1(50, 0x7d, 0, 1), // flat_load_dword + EncodeFlat0(0x0d, 1, 8), + EncodeFlat1(52, 0x7d, 0, 1), // scratch_load_dwordx2 + EncodeFlat0(0x0e, 2, 12), + EncodeFlat1(56, 0x7d, 0, 1), // global_load_dwordx4 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.flat_memory_base = 0; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_load_dwordx4"), - "new decoder did not decode SMEM x4 load"); - Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx4"), - "new decoder did not decode scalar-buffer x4 load"); - Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx2 vcc_lo"), - "new decoder did not decode scalar-buffer x2 load into VCC"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx2"), - "new decoder did not decode buffer x2 load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx3"), - "new decoder did not decode buffer x3 load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx4"), - "new decoder did not decode buffer x4 load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dwordx4"), - "new decoder did not decode buffer x4 store"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ubyte"), - "new decoder did not decode buffer ubyte load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ushort"), - "new decoder did not decode buffer ushort load"); - Check(Common::ContainsStr(result.decoded_dump, "flat_load_dword"), - "new decoder did not decode flat dword load"); - Check(Common::ContainsStr(result.decoded_dump, "segment=1"), - "new decoder did not preserve scratch segment metadata"); - Check(Common::ContainsStr(result.decoded_dump, "segment=2"), - "new decoder did not preserve global segment metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), - "new decoder did not expose width metadata for wide memory ops"); - Check(Common::ContainsStr(result.ir_dump, "SLoadDword s4"), - "s_load_dwordx4 did not expand to first scalar dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "SLoadDword s7"), - "s_load_dwordx4 did not expand to last scalar dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s8"), - "s_buffer_load_dwordx4 did not expand to first scalar-buffer IR load"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s11"), - "s_buffer_load_dwordx4 did not expand to last scalar-buffer IR load"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_lo"), - "s_buffer_load_dwordx2 did not expand to VCC low dword"); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), - "s_buffer_load_dwordx2 did not expand to VCC high dword"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v20"), - "buffer_load_dwordx2 did not expand to dword IR loads"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"), - "buffer_load_dwordx4 did not expand to the last dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v40"), - "buffer_store_dwordx4 did not expand to first dword IR store"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"), - "buffer_store_dwordx4 did not expand to last dword IR store"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadUbyte v44"), - "buffer_load_ubyte did not lower to sub-dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadUshort v45"), - "buffer_load_ushort did not lower to sub-dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "FlatLoadDword v50"), - "flat_load_dword did not lower to flat dword IR load"); - Check(Common::ContainsStr(result.ir_dump, "scratch"), - "scratch segment load did not lower with scratch metadata"); - Check(Common::ContainsStr(result.ir_dump, "global"), - "global segment load did not lower with global metadata"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_load_dwordx4"), + "new decoder did not decode SMEM x4 load"); + Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx4"), + "new decoder did not decode scalar-buffer x4 load"); + Check( + Common::ContainsStr(result.decoded_dump, "s_buffer_load_dwordx2 vcc_lo"), + "new decoder did not decode scalar-buffer x2 load into VCC"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx2"), + "new decoder did not decode buffer x2 load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx3"), + "new decoder did not decode buffer x3 load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dwordx4"), + "new decoder did not decode buffer x4 load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_dwordx4"), + "new decoder did not decode buffer x4 store"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ubyte"), + "new decoder did not decode buffer ubyte load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_ushort"), + "new decoder did not decode buffer ushort load"); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_dword"), + "new decoder did not decode flat dword load"); + Check(Common::ContainsStr(result.decoded_dump, "segment=1"), + "new decoder did not preserve scratch segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "segment=2"), + "new decoder did not preserve global segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "new decoder did not expose width metadata for wide memory ops"); + Check(Common::ContainsStr(result.ir_dump, "SLoadDword s4"), + "s_load_dwordx4 did not expand to first scalar dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "SLoadDword s7"), + "s_load_dwordx4 did not expand to last scalar dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s8"), + "s_buffer_load_dwordx4 did not expand to first scalar-buffer IR load"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword s11"), + "s_buffer_load_dwordx4 did not expand to last scalar-buffer IR load"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_lo"), + "s_buffer_load_dwordx2 did not expand to VCC low dword"); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), + "s_buffer_load_dwordx2 did not expand to VCC high dword"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v20"), + "buffer_load_dwordx2 did not expand to dword IR loads"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v31"), + "buffer_load_dwordx4 did not expand to the last dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v40"), + "buffer_store_dwordx4 did not expand to first dword IR store"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v43"), + "buffer_store_dwordx4 did not expand to last dword IR store"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadUbyte v44"), + "buffer_load_ubyte did not lower to sub-dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadUshort v45"), + "buffer_load_ushort did not lower to sub-dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadDword v50"), + "flat_load_dword did not lower to flat dword IR load"); + Check(Common::ContainsStr(result.ir_dump, "scratch"), + "scratch segment load did not lower with scratch metadata"); + Check(Common::ContainsStr(result.ir_dump, "global"), + "global segment load did not lower with global metadata"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerBufferSignedLoadLowering() { - const uint32_t shader[] = { - EncodeMubuf0(0x09, 3), EncodeMubuf1(46, 0, 1), // buffer_load_sbyte - EncodeMubuf0(0x0b, 6), EncodeMubuf1(47, 0, 1), // buffer_load_sshort - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMubuf0(0x09, 3), EncodeMubuf1(46, 0, 1), // buffer_load_sbyte + EncodeMubuf0(0x0b, 6), EncodeMubuf1(47, 0, 1), // buffer_load_sshort + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sbyte"), - "new decoder did not decode buffer signed byte load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sshort"), - "new decoder did not decode buffer signed short load"); - Check(Common::ContainsStr(result.decoded_dump, "bits=8"), - "signed byte buffer load did not preserve bit width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "bits=16"), - "signed short buffer load did not preserve bit width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "signed=1"), - "signed buffer loads did not preserve signed metadata"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadSbyte v46"), - "buffer_load_sbyte did not lower to signed byte IR load"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadSshort v47"), - "buffer_load_sshort did not lower to signed short IR load"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 195), - "SPIR-V binary does not contain OpShiftRightArithmetic for sign extension"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical for sign extension"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sbyte"), + "new decoder did not decode buffer signed byte load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sshort"), + "new decoder did not decode buffer signed short load"); + Check(Common::ContainsStr(result.decoded_dump, "bits=8"), + "signed byte buffer load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "bits=16"), + "signed short buffer load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "signed=1"), + "signed buffer loads did not preserve signed metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadSbyte v46"), + "buffer_load_sbyte did not lower to signed byte IR load"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadSshort v47"), + "buffer_load_sshort did not lower to signed short IR load"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic for sign " + "extension"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for sign extension"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerBufferSubDwordStoreLowering() { - const uint32_t shader[] = { - EncodeMubuf0(0x18, 2), EncodeMubuf1(48, 0, 1), // buffer_store_byte - EncodeMubuf0(0x1a, 4), EncodeMubuf1(49, 0, 1), // buffer_store_short - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMubuf0(0x18, 2), EncodeMubuf1(48, 0, 1), // buffer_store_byte + EncodeMubuf0(0x1a, 4), EncodeMubuf1(49, 0, 1), // buffer_store_short + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_byte"), - "new decoder did not decode buffer byte store"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_short"), - "new decoder did not decode buffer short store"); - Check(Common::ContainsStr(result.decoded_dump, "bits=8"), - "buffer byte store did not preserve bit width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "bits=16"), - "buffer short store did not preserve bit width metadata"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreByte null, v48"), - "buffer_store_byte did not lower to byte store IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreShort null, v49"), - "buffer_store_short did not lower to short store IR"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad for sub-dword store RMW"); - Check(SpirvContainsOpcode(result.spirv, 62), - "SPIR-V binary does not contain OpStore for sub-dword store RMW"); - Check(SpirvContainsOpcode(result.spirv, 197), "SPIR-V binary does not contain OpBitwiseOr"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - Check(SpirvContainsOpcode(result.spirv, 200), "SPIR-V binary does not contain OpNot"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_byte"), + "new decoder did not decode buffer byte store"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_short"), + "new decoder did not decode buffer short store"); + Check(Common::ContainsStr(result.decoded_dump, "bits=8"), + "buffer byte store did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "bits=16"), + "buffer short store did not preserve bit width metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreByte null, v48"), + "buffer_store_byte did not lower to byte store IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreShort null, v49"), + "buffer_store_short did not lower to short store IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for sub-dword store RMW"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore for sub-dword store RMW"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerMubufFormatLowering() { - const uint32_t shader[] = { - EncodeMubuf0(0x00, 4), - EncodeMubuf1(84, 0, 1), // buffer_load_format_x - EncodeMubuf0(0x01, 8), - EncodeMubuf1(88, 0, 1), // buffer_load_format_xy - EncodeMubuf0(0x02, 12), - EncodeMubuf1(92, 0, 1), // buffer_load_format_xyz - EncodeMubuf0(0x03, 16), - EncodeMubuf1(96, 0, 1), // buffer_load_format_xyzw - EncodeMubuf0(0x04, 20), - EncodeMubuf1(100, 0, 1), // buffer_store_format_x - EncodeMubuf0(0x05, 24), - EncodeMubuf1(104, 0, 1), // buffer_store_format_xy - EncodeMubuf0(0x06, 28), - EncodeMubuf1(108, 0, 1), // buffer_store_format_xyz - EncodeMubuf0(0x07, 32), - EncodeMubuf1(112, 0, 1), // buffer_store_format_xyzw - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMubuf0(0x00, 4), + EncodeMubuf1(84, 0, 1), // buffer_load_format_x + EncodeMubuf0(0x01, 8), + EncodeMubuf1(88, 0, 1), // buffer_load_format_xy + EncodeMubuf0(0x02, 12), + EncodeMubuf1(92, 0, 1), // buffer_load_format_xyz + EncodeMubuf0(0x03, 16), + EncodeMubuf1(96, 0, 1), // buffer_load_format_xyzw + EncodeMubuf0(0x04, 20), + EncodeMubuf1(100, 0, 1), // buffer_store_format_x + EncodeMubuf0(0x05, 24), + EncodeMubuf1(104, 0, 1), // buffer_store_format_xy + EncodeMubuf0(0x06, 28), + EncodeMubuf1(108, 0, 1), // buffer_store_format_xyz + EncodeMubuf0(0x07, 32), + EncodeMubuf1(112, 0, 1), // buffer_store_format_xyzw + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_x"), - "new decoder did not decode MUBUF format-x load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_xyzw"), - "new decoder did not decode MUBUF format-xyzw load"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), - "new decoder did not decode MUBUF format-x store"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_xyzw"), - "new decoder did not decode MUBUF format-xyzw store"); - Check(Common::ContainsStr(result.decoded_dump, "typed=0 formatted=1"), - "MUBUF format decode did not preserve formatted non-typed metadata"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v84"), - "MUBUF format-x load did not lower through shared buffer load IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v99"), - "MUBUF format-xyzw load did not expand to the last shared dword load"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v100"), - "MUBUF format-x store did not lower through shared buffer store IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v115"), - "MUBUF format-xyzw store did not expand to the last shared dword store"); - Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), - "MUBUF format metadata did not survive into IR"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_x"), + "new decoder did not decode MUBUF format-x load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_format_xyzw"), + "new decoder did not decode MUBUF format-xyzw load"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), + "new decoder did not decode MUBUF format-x store"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_xyzw"), + "new decoder did not decode MUBUF format-xyzw store"); + Check(Common::ContainsStr(result.decoded_dump, "typed=0 formatted=1"), + "MUBUF format decode did not preserve formatted non-typed metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v84"), + "MUBUF format-x load did not lower through shared buffer load IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v99"), + "MUBUF format-xyzw load did not expand to the last shared dword load"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v100"), + "MUBUF format-x store did not lower through shared buffer store IR"); + Check( + Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v115"), + "MUBUF format-xyzw store did not expand to the last shared dword store"); + Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), + "MUBUF format metadata did not survive into IR"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() { - const uint32_t shader[] = { - EncodeMubuf0(0x04), - EncodeMubuf1(0, 0, 1), // buffer_store_format_x - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMubuf0(0x04), + EncodeMubuf1(0, 0, 1), // buffer_store_format_x + 0xbf810000u, + }; - std::array user_data {}; - user_data[1] = 4u << 16u; - user_data[2] = 5u; - user_data[3] = static_cast(Prospero::BufferFormat::k32UInt) << 12u; + std::array user_data{}; + user_data[1] = 4u << 16u; + user_data[2] = 5u; + user_data[3] = static_cast(Prospero::BufferFormat::k32UInt) << 12u; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(result.resources.buffers.size() == 1 && result.resources.buffers[0].dwords[2] == 5u, - "formatted store test did not preserve descriptor NumRecords"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), - "formatted store regression did not decode buffer_store_format_x"); - Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), - "formatted store regression did not preserve formatted metadata"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(result.resources.buffers.size() == 1 && + result.resources.buffers[0].dwords[2] == 5u, + "formatted store test did not preserve descriptor NumRecords"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_store_format_x"), + "formatted store regression did not decode buffer_store_format_x"); + Check(Common::ContainsStr(result.ir_dump, "typed=0 formatted=1"), + "formatted store regression did not preserve formatted metadata"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(source, "OpArrayLength"), - "formatted store SPIR-V lacks runtime storage-buffer bounds check"); - Check(!SpirvSourceHasInstructionUsing(source, "OpULessThan", "%uint_5"), - "formatted store SPIR-V baked descriptor NumRecords into a store guard"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "OpArrayLength"), + "formatted store SPIR-V lacks runtime storage-buffer bounds check"); + Check( + !SpirvSourceHasInstructionUsing(source, "OpULessThan", "%uint_5"), + "formatted store SPIR-V baked descriptor NumRecords into a store guard"); } void TestNewShaderRecompilerTypedBufferLowering() { - const uint32_t shader[] = { - EncodeMtbuf0(0x00, 14, 7, 4), - EncodeMtbuf1(0x00, 60, 0, 1), // tbuffer_load_format_x - EncodeMtbuf0(0x01, 13, 7, 8), - EncodeMtbuf1(0x01, 61, 0, 1), // tbuffer_load_format_xy - EncodeMtbuf0(0x02, 13, 4, 12), - EncodeMtbuf1(0x02, 64, 0, 1), // tbuffer_load_format_xyz - EncodeMtbuf0(0x03, 14, 7, 16, true, true), - EncodeMtbuf1(0x03, 68, 0, 1), // tbuffer_load_format_xyzw - EncodeMtbuf0(0x04, 14, 7, 20), - EncodeMtbuf1(0x04, 72, 0, 1), // tbuffer_store_format_x - EncodeMtbuf0(0x05, 13, 7, 24), - EncodeMtbuf1(0x05, 76, 0, 1), // tbuffer_store_format_xy - EncodeMtbuf0(0x07, 14, 7, 28), - EncodeMtbuf1(0x07, 80, 0, 1), // tbuffer_store_format_xyzw - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMtbuf0(0x00, 14, 7, 4), + EncodeMtbuf1(0x00, 60, 0, 1), // tbuffer_load_format_x + EncodeMtbuf0(0x01, 13, 7, 8), + EncodeMtbuf1(0x01, 61, 0, 1), // tbuffer_load_format_xy + EncodeMtbuf0(0x02, 13, 4, 12), + EncodeMtbuf1(0x02, 64, 0, 1), // tbuffer_load_format_xyz + EncodeMtbuf0(0x03, 14, 7, 16, true, true), + EncodeMtbuf1(0x03, 68, 0, 1), // tbuffer_load_format_xyzw + EncodeMtbuf0(0x04, 14, 7, 20), + EncodeMtbuf1(0x04, 72, 0, 1), // tbuffer_store_format_x + EncodeMtbuf0(0x05, 13, 7, 24), + EncodeMtbuf1(0x05, 76, 0, 1), // tbuffer_store_format_xy + EncodeMtbuf0(0x07, 14, 7, 28), + EncodeMtbuf1(0x07, 80, 0, 1), // tbuffer_store_format_xyzw + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_x"), - "new decoder did not decode typed buffer format-x load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xy"), - "new decoder did not decode typed buffer format-xy load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyz"), - "new decoder did not decode typed buffer format-xyz load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyzw"), - "new decoder did not decode typed buffer format-xyzw load"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_x"), - "new decoder did not decode typed buffer format-x store"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xy"), - "new decoder did not decode typed buffer format-xy store"); - Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xyzw"), - "new decoder did not decode typed buffer format-xyzw store"); - Check(Common::ContainsStr(result.decoded_dump, "dfmt=14 nfmt=7"), - "MTBUF decode did not expose dfmt/nfmt metadata"); - Check(Common::ContainsStr(result.decoded_dump, "typed=1"), - "MTBUF decode did not preserve typed metadata"); - Check(Common::ContainsStr(result.decoded_dump, "offen=1"), - "MTBUF decode did not preserve offen metadata"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v60"), - "typed buffer x load did not lower through shared buffer load IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v71"), - "typed buffer xyzw load did not expand to last shared dword load"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v72"), - "typed buffer x store did not lower through shared buffer store IR"); - Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v83"), - "typed buffer xyzw store did not expand to last shared dword store"); - Check(Common::ContainsStr(result.ir_dump, "typed=1"), - "typed buffer metadata did not survive into IR"); - Check(Common::ContainsStr(result.ir_dump, "dfmt=13 nfmt=7"), - "typed buffer format metadata did not survive into IR"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_x"), + "new decoder did not decode typed buffer format-x load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xy"), + "new decoder did not decode typed buffer format-xy load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyz"), + "new decoder did not decode typed buffer format-xyz load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_load_format_xyzw"), + "new decoder did not decode typed buffer format-xyzw load"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_x"), + "new decoder did not decode typed buffer format-x store"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xy"), + "new decoder did not decode typed buffer format-xy store"); + Check(Common::ContainsStr(result.decoded_dump, "tbuffer_store_format_xyzw"), + "new decoder did not decode typed buffer format-xyzw store"); + Check(Common::ContainsStr(result.decoded_dump, "dfmt=14 nfmt=7"), + "MTBUF decode did not expose dfmt/nfmt metadata"); + Check(Common::ContainsStr(result.decoded_dump, "typed=1"), + "MTBUF decode did not preserve typed metadata"); + Check(Common::ContainsStr(result.decoded_dump, "offen=1"), + "MTBUF decode did not preserve offen metadata"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v60"), + "typed buffer x load did not lower through shared buffer load IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferLoadDword v71"), + "typed buffer xyzw load did not expand to last shared dword load"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v72"), + "typed buffer x store did not lower through shared buffer store IR"); + Check(Common::ContainsStr(result.ir_dump, "BufferStoreDword null, v83"), + "typed buffer xyzw store did not expand to last shared dword store"); + Check(Common::ContainsStr(result.ir_dump, "typed=1"), + "typed buffer metadata did not survive into IR"); + Check(Common::ContainsStr(result.ir_dump, "dfmt=13 nfmt=7"), + "typed buffer format metadata did not survive into IR"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerFlatOldBackedLowering() { - const uint32_t shader[] = { - EncodeFlat0(0x08, 0, 4), - EncodeFlat1(9, 0x7d, 0, 1), // flat_load_ubyte - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeFlat0(0x08, 0, 4), + EncodeFlat1(9, 0x7d, 0, 1), // flat_load_ubyte + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.flat_memory_base = 0; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "flat_load_ubyte"), - "new decoder did not decode old-backed FLAT ubyte load"); - Check(Common::ContainsStr(result.ir_dump, "FlatLoadUbyte v9"), - "old-backed FLAT ubyte load did not lower to IR"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_ubyte"), + "new decoder did not decode old-backed FLAT ubyte load"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadUbyte v9"), + "old-backed FLAT ubyte load did not lower to IR"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerUnbasedFlatRequiresTranslator() { - const uint32_t shader[] = { - EncodeFlat0(0x0c, 0, 0), - EncodeFlat1(0, 0x7d, 0, 1), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeFlat0(0x0c, 0, 0), + EncodeFlat1(0, 0x7d, 0, 1), + EncodeFlat0(0x1c, 0, 0), + EncodeFlat1(0, 0x7d, 0, 2), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; - ShaderRecompiler::CompileResult result; - std::string error; - Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && - Common::ContainsStr(error, "requires runtime guest-address translation"), - "unbased FLAT compiled without an explicit translator"); + ShaderRecompiler::CompileResult result; + std::string error; + Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && + Common::ContainsStr(error, + "requires runtime guest-address translation"), + "unbased FLAT compiled without an explicit translator"); } void TestNewShaderRecompilerFlatSignedLoadLowering() { - const uint32_t shader[] = { - EncodeFlat0(0x09, 0, 4), - EncodeFlat1(10, 0x7d, 0, 1), // flat_load_sbyte - EncodeFlat0(0x0b, 1, 8), - EncodeFlat1(11, 0x7d, 0, 1), // scratch_load_sshort - EncodeFlat0(0x09, 2, 12), - EncodeFlat1(12, 0x7d, 0, 1), // global_load_sbyte - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeFlat0(0x09, 0, 4), + EncodeFlat1(10, 0x7d, 0, 1), // flat_load_sbyte + EncodeFlat0(0x0b, 1, 8), + EncodeFlat1(11, 0x7d, 0, 1), // scratch_load_sshort + EncodeFlat0(0x09, 2, 12), + EncodeFlat1(12, 0x7d, 0, 1), // global_load_sbyte + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.flat_memory_base = 0; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "flat_load_sbyte"), - "new decoder did not decode flat signed byte load"); - Check(Common::ContainsStr(result.decoded_dump, "flat_load_sshort"), - "new decoder did not decode flat signed short load"); - Check(Common::ContainsStr(result.decoded_dump, "bits=8"), - "signed byte flat load did not preserve bit width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "bits=16"), - "signed short flat load did not preserve bit width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "signed=1"), - "signed flat loads did not preserve signed metadata"); - Check(Common::ContainsStr(result.ir_dump, "FlatLoadSbyte v10"), - "flat_load_sbyte did not lower to signed byte IR load"); - Check(Common::ContainsStr(result.ir_dump, "FlatLoadSshort v11"), - "flat_load_sshort did not lower to signed short IR load"); - Check(Common::ContainsStr(result.ir_dump, "scratch"), - "signed scratch load did not preserve scratch metadata"); - Check(Common::ContainsStr(result.ir_dump, "global"), - "signed global load did not preserve global metadata"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 195), - "SPIR-V binary does not contain OpShiftRightArithmetic for sign extension"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical for sign extension"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_sbyte"), + "new decoder did not decode flat signed byte load"); + Check(Common::ContainsStr(result.decoded_dump, "flat_load_sshort"), + "new decoder did not decode flat signed short load"); + Check(Common::ContainsStr(result.decoded_dump, "bits=8"), + "signed byte flat load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "bits=16"), + "signed short flat load did not preserve bit width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "signed=1"), + "signed flat loads did not preserve signed metadata"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadSbyte v10"), + "flat_load_sbyte did not lower to signed byte IR load"); + Check(Common::ContainsStr(result.ir_dump, "FlatLoadSshort v11"), + "flat_load_sshort did not lower to signed short IR load"); + Check(Common::ContainsStr(result.ir_dump, "scratch"), + "signed scratch load did not preserve scratch metadata"); + Check(Common::ContainsStr(result.ir_dump, "global"), + "signed global load did not preserve global metadata"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic for sign " + "extension"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical for sign extension"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerFlatStoreLowering() { - const uint32_t shader[] = { - EncodeFlat0(0x18, 0, 2), - EncodeFlat1(0, 0x7d, 72, 1), // flat_store_byte - EncodeFlat0(0x1a, 1, 6), - EncodeFlat1(0, 0x7d, 73, 1), // scratch_store_short - EncodeFlat0(0x1c, 0, 4), - EncodeFlat1(0, 0x7d, 60, 1), // flat_store_dword - EncodeFlat0(0x1d, 1, 8), - EncodeFlat1(0, 0x7d, 61, 1), // scratch_store_dwordx2 - EncodeFlat0(0x1f, 2, 12), - EncodeFlat1(0, 0x7d, 64, 1), // global_store_dwordx3 - EncodeFlat0(0x1e, 2, 16), - EncodeFlat1(0, 0x7d, 68, 1), // global_store_dwordx4 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeFlat0(0x18, 0, 2), + EncodeFlat1(0, 0x7d, 72, 1), // flat_store_byte + EncodeFlat0(0x1a, 1, 6), + EncodeFlat1(0, 0x7d, 73, 1), // scratch_store_short + EncodeFlat0(0x1c, 0, 4), + EncodeFlat1(0, 0x7d, 60, 1), // flat_store_dword + EncodeFlat0(0x1d, 1, 8), + EncodeFlat1(0, 0x7d, 61, 1), // scratch_store_dwordx2 + EncodeFlat0(0x1f, 2, 12), + EncodeFlat1(0, 0x7d, 64, 1), // global_store_dwordx3 + EncodeFlat0(0x1e, 2, 16), + EncodeFlat1(0, 0x7d, 68, 1), // global_store_dwordx4 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.flat_memory_base = 0; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.flat_memory_base = 0; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "flat_store_byte"), - "new decoder did not decode flat byte store"); - Check(Common::ContainsStr(result.decoded_dump, "flat_store_short"), - "new decoder did not decode flat short store"); - Check(Common::ContainsStr(result.decoded_dump, "flat_store_dword"), - "new decoder did not decode flat dword store"); - Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx2"), - "new decoder did not decode flat dwordx2 store"); - Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx3"), - "new decoder did not decode flat dwordx3 store"); - Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx4"), - "new decoder did not decode flat dwordx4 store"); - Check(Common::ContainsStr(result.decoded_dump, "segment=1"), - "flat store decode did not preserve scratch segment metadata"); - Check(Common::ContainsStr(result.decoded_dump, "segment=2"), - "flat store decode did not preserve global segment metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"), - "flat byte store decode did not expose byte-width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"), - "flat short store decode did not expose short-width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), - "flat store decode did not expose wide width metadata"); - Check(Common::ContainsStr(result.ir_dump, "FlatStoreByte null, v72"), - "flat_store_byte did not lower through shared flat store IR"); - Check(Common::ContainsStr(result.ir_dump, "FlatStoreShort null, v73"), - "flat_store_short did not lower through shared flat store IR"); - Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v60"), - "flat_store_dword did not lower through shared flat store IR"); - Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v62"), - "scratch_store_dwordx2 did not expand to the last dword store"); - Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v66"), - "global_store_dwordx3 did not expand to the last dword store"); - Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v71"), - "global_store_dwordx4 did not expand to the last dword store"); - Check(Common::ContainsStr(result.ir_dump, "flat"), - "flat store IR did not preserve flat resource metadata"); - Check(Common::ContainsStr(result.ir_dump, "scratch"), - "flat store IR did not preserve scratch resource metadata"); - Check(Common::ContainsStr(result.ir_dump, "global"), - "flat store IR did not preserve global resource metadata"); - Check(SpirvContainsOpcode(result.spirv, 61), - "SPIR-V binary does not contain OpLoad for sub-dword flat store merge"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 197), - "SPIR-V binary does not contain OpBitwiseOr for sub-dword flat store merge"); - Check(SpirvContainsOpcode(result.spirv, 200), - "SPIR-V binary does not contain OpNot for sub-dword flat store mask"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_byte"), + "new decoder did not decode flat byte store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_short"), + "new decoder did not decode flat short store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dword"), + "new decoder did not decode flat dword store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx2"), + "new decoder did not decode flat dwordx2 store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx3"), + "new decoder did not decode flat dwordx3 store"); + Check(Common::ContainsStr(result.decoded_dump, "flat_store_dwordx4"), + "new decoder did not decode flat dwordx4 store"); + Check(Common::ContainsStr(result.decoded_dump, "segment=1"), + "flat store decode did not preserve scratch segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "segment=2"), + "flat store decode did not preserve global segment metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"), + "flat byte store decode did not expose byte-width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"), + "flat short store decode did not expose short-width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "flat store decode did not expose wide width metadata"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreByte null, v72"), + "flat_store_byte did not lower through shared flat store IR"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreShort null, v73"), + "flat_store_short did not lower through shared flat store IR"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v60"), + "flat_store_dword did not lower through shared flat store IR"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v62"), + "scratch_store_dwordx2 did not expand to the last dword store"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v66"), + "global_store_dwordx3 did not expand to the last dword store"); + Check(Common::ContainsStr(result.ir_dump, "FlatStoreDword null, v71"), + "global_store_dwordx4 did not expand to the last dword store"); + Check(Common::ContainsStr(result.ir_dump, "flat"), + "flat store IR did not preserve flat resource metadata"); + Check(Common::ContainsStr(result.ir_dump, "scratch"), + "flat store IR did not preserve scratch resource metadata"); + Check(Common::ContainsStr(result.ir_dump, "global"), + "flat store IR did not preserve global resource metadata"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad for sub-dword flat store merge"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 197), + "SPIR-V binary does not contain OpBitwiseOr for sub-dword flat store " + "merge"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot for sub-dword flat store mask"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerAtomicLowering() { - const uint32_t shader[] = { - EncodeMubuf0(0x30, 4, true, true), - EncodeMubuf1(0, 0, 1), // buffer_atomic_swap - EncodeMubuf0(0x32, 8, true, true), - EncodeMubuf1(1, 0, 1), // buffer_atomic_add - EncodeMubuf0(0x33, 12, true, true), - EncodeMubuf1(7, 0, 1), // buffer_atomic_sub - EncodeMubuf0(0x35, 16, true, true), - EncodeMubuf1(8, 0, 1), // buffer_atomic_smin - EncodeMubuf0(0x36, 20, true, true), - EncodeMubuf1(2, 0, 1), // buffer_atomic_umin - EncodeMubuf0(0x37, 24, true, true), - EncodeMubuf1(9, 0, 1), // buffer_atomic_smax - EncodeMubuf0(0x38, 28, true, true), - EncodeMubuf1(3, 0, 1), // buffer_atomic_umax - EncodeMubuf0(0x39, 32, true, true), - EncodeMubuf1(4, 0, 1), // buffer_atomic_and - EncodeMubuf0(0x3a, 36, true, true), - EncodeMubuf1(5, 0, 1), // buffer_atomic_or - EncodeMubuf0(0x3b, 40, true, true), - EncodeMubuf1(6, 0, 1), // buffer_atomic_xor - 0xe0fc0000u, - 0x80010000u, // exact buffer_atomic_fmin v0, s[4:7], 0 (GLC=0) - 0xe100000cu, - 0x80010300u, // exact failing buffer_atomic_fmax v3, s[4:7], 12 (GLC=0) - EncodeDs0(0x00), - EncodeDs1(0, 2, 1), // ds_add_u32 - EncodeDs0(0x01), - EncodeDs1(0, 10, 1), // ds_sub_u32 - EncodeDs0(0x05), - EncodeDs1(0, 11, 1), // ds_min_i32 - EncodeDs0(0x06), - EncodeDs1(0, 12, 1), // ds_max_i32 - EncodeDs0(0x09), - EncodeDs1(0, 13, 1), // ds_and_b32 - EncodeDs0(0x0b), - EncodeDs1(0, 14, 1), // ds_xor_b32 - EncodeDs0(0x20), - EncodeDs1(15, 2, 1), // ds_add_rtn_u32 - EncodeDs0(0x21), - EncodeDs1(16, 10, 1), // ds_sub_rtn_u32 - EncodeDs0(0x25), - EncodeDs1(17, 11, 1), // ds_min_rtn_i32 - EncodeDs0(0x26), - EncodeDs1(18, 12, 1), // ds_max_rtn_i32 - EncodeDs0(0x27), - EncodeDs1(19, 2, 1), // ds_min_rtn_u32 - EncodeDs0(0x28), - EncodeDs1(20, 2, 1), // ds_max_rtn_u32 - EncodeDs0(0x29), - EncodeDs1(21, 13, 1), // ds_and_rtn_b32 - EncodeDs0(0x2a), - EncodeDs1(22, 2, 1), // ds_or_rtn_b32 - EncodeDs0(0x2b), - EncodeDs1(23, 14, 1), // ds_xor_rtn_b32 - EncodeDs0(0x2d), - EncodeDs1(24, 15, 1), // ds_wrxchg_rtn_b32 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeMubuf0(0x30, 4, true, true), + EncodeMubuf1(0, 0, 1), // buffer_atomic_swap + EncodeMubuf0(0x32, 8, true, true), + EncodeMubuf1(1, 0, 1), // buffer_atomic_add + EncodeMubuf0(0x33, 12, true, true), + EncodeMubuf1(7, 0, 1), // buffer_atomic_sub + EncodeMubuf0(0x35, 16, true, true), + EncodeMubuf1(8, 0, 1), // buffer_atomic_smin + EncodeMubuf0(0x36, 20, true, true), + EncodeMubuf1(2, 0, 1), // buffer_atomic_umin + EncodeMubuf0(0x37, 24, true, true), + EncodeMubuf1(9, 0, 1), // buffer_atomic_smax + EncodeMubuf0(0x38, 28, true, true), + EncodeMubuf1(3, 0, 1), // buffer_atomic_umax + EncodeMubuf0(0x39, 32, true, true), + EncodeMubuf1(4, 0, 1), // buffer_atomic_and + EncodeMubuf0(0x3a, 36, true, true), + EncodeMubuf1(5, 0, 1), // buffer_atomic_or + EncodeMubuf0(0x3b, 40, true, true), + EncodeMubuf1(6, 0, 1), // buffer_atomic_xor + 0xe0fc0000u, + 0x80010000u, // exact buffer_atomic_fmin v0, s[4:7], 0 (GLC=0) + 0xe100000cu, + 0x80010300u, // exact failing buffer_atomic_fmax v3, s[4:7], 12 (GLC=0) + EncodeDs0(0x00), + EncodeDs1(0, 2, 1), // ds_add_u32 + EncodeDs0(0x01), + EncodeDs1(0, 10, 1), // ds_sub_u32 + EncodeDs0(0x05), + EncodeDs1(0, 11, 1), // ds_min_i32 + EncodeDs0(0x06), + EncodeDs1(0, 12, 1), // ds_max_i32 + EncodeDs0(0x09), + EncodeDs1(0, 13, 1), // ds_and_b32 + EncodeDs0(0x0b), + EncodeDs1(0, 14, 1), // ds_xor_b32 + EncodeDs0(0x20), + EncodeDs1(15, 2, 1), // ds_add_rtn_u32 + EncodeDs0(0x21), + EncodeDs1(16, 10, 1), // ds_sub_rtn_u32 + EncodeDs0(0x25), + EncodeDs1(17, 11, 1), // ds_min_rtn_i32 + EncodeDs0(0x26), + EncodeDs1(18, 12, 1), // ds_max_rtn_i32 + EncodeDs0(0x27), + EncodeDs1(19, 2, 1), // ds_min_rtn_u32 + EncodeDs0(0x28), + EncodeDs1(20, 2, 1), // ds_max_rtn_u32 + EncodeDs0(0x29), + EncodeDs1(21, 13, 1), // ds_and_rtn_b32 + EncodeDs0(0x2a), + EncodeDs1(22, 2, 1), // ds_or_rtn_b32 + EncodeDs0(0x2b), + EncodeDs1(23, 14, 1), // ds_xor_rtn_b32 + EncodeDs0(0x2d), + EncodeDs1(24, 15, 1), // ds_wrxchg_rtn_b32 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_swap"), - "new decoder did not decode buffer atomic swap"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), - "new decoder did not decode buffer atomic add"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_sub"), - "new decoder did not decode buffer atomic sub"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_smin"), - "new decoder did not decode buffer atomic signed min"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_smax"), - "new decoder did not decode buffer atomic signed max"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_xor"), - "new decoder did not decode buffer atomic xor"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_fmin"), - "new decoder did not decode buffer atomic float min"); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_fmax"), - "new decoder did not decode buffer atomic float max"); - Check(Common::ContainsStr(result.decoded_dump, "ds_add_u32"), - "new decoder did not decode DS atomic add"); - Check(Common::ContainsStr(result.decoded_dump, "ds_sub_u32"), - "new decoder did not decode DS atomic sub"); - Check(Common::ContainsStr(result.decoded_dump, "ds_min_i32"), - "new decoder did not decode DS signed min"); - Check(Common::ContainsStr(result.decoded_dump, "ds_xor_rtn_b32"), - "new decoder did not decode DS xor-return"); - Check(Common::ContainsStr(result.decoded_dump, "ds_wrxchg_rtn_b32"), - "new decoder did not decode DS write-exchange-return"); - Check(Common::ContainsStr(result.decoded_dump, "ds_add_rtn_u32"), - "new decoder did not decode DS atomic add-return"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSwapU32 v0"), - "buffer atomic swap did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v1"), - "buffer atomic add did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 v7"), - "buffer atomic sub did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 v8"), - "buffer atomic signed min did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v2"), - "buffer atomic umin did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 v9"), - "buffer atomic signed max did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v3"), - "buffer atomic umax did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v4"), - "buffer atomic and did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v5"), - "buffer atomic or did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v6"), - "buffer atomic xor did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicFMinF32 null, v0"), - "buffer atomic float min did not lower to IR without a GLC return"); - Check(Common::ContainsStr(result.ir_dump, "AtomicFMaxF32 null, v3"), - "buffer atomic float max did not lower to IR without a GLC return"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 null, v2"), - "DS no-return atomic add did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 null, v10"), - "DS no-return atomic sub did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 null, v11"), - "DS no-return signed min did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 null, v12"), - "DS no-return signed max did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 null, v13"), - "DS no-return and did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 null, v14"), - "DS no-return xor did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v15"), - "DS add-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 v16"), - "DS sub-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 v17"), - "DS signed min-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 v18"), - "DS signed max-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v19"), - "DS unsigned min-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v20"), - "DS unsigned max-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v21"), - "DS and-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v22"), - "DS or-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v23"), - "DS xor-return did not lower to IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicSwapU32 v24"), - "DS write-exchange-return did not lower to shared atomic swap IR"); - Check(SpirvContainsOpcode(result.spirv, 229), - "SPIR-V binary does not contain OpAtomicExchange"); - Check(SpirvContainsOpcode(result.spirv, 234), "SPIR-V binary does not contain OpAtomicIAdd"); - Check(SpirvContainsOpcode(result.spirv, 235), "SPIR-V binary does not contain OpAtomicISub"); - Check(SpirvContainsOpcode(result.spirv, 236), "SPIR-V binary does not contain OpAtomicSMin"); - Check(SpirvContainsOpcode(result.spirv, 237), "SPIR-V binary does not contain OpAtomicUMin"); - Check(SpirvContainsOpcode(result.spirv, 238), "SPIR-V binary does not contain OpAtomicSMax"); - Check(SpirvContainsOpcode(result.spirv, 239), "SPIR-V binary does not contain OpAtomicUMax"); - Check(SpirvContainsOpcode(result.spirv, 240), "SPIR-V binary does not contain OpAtomicAnd"); - Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr"); - Check(SpirvContainsOpcode(result.spirv, 242), "SPIR-V binary does not contain OpAtomicXor"); - Check(SpirvContainsOpcode(result.spirv, 230), - "SPIR-V binary does not contain OpAtomicCompareExchange for float min"); - Check(SpirvContainsOpcode(result.spirv, 225), - "buffer atomic SPIR-V binary does not contain OpMemoryBarrier"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_swap"), + "new decoder did not decode buffer atomic swap"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), + "new decoder did not decode buffer atomic add"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_sub"), + "new decoder did not decode buffer atomic sub"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_smin"), + "new decoder did not decode buffer atomic signed min"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_smax"), + "new decoder did not decode buffer atomic signed max"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_xor"), + "new decoder did not decode buffer atomic xor"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_fmin"), + "new decoder did not decode buffer atomic float min"); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_fmax"), + "new decoder did not decode buffer atomic float max"); + Check(Common::ContainsStr(result.decoded_dump, "ds_add_u32"), + "new decoder did not decode DS atomic add"); + Check(Common::ContainsStr(result.decoded_dump, "ds_sub_u32"), + "new decoder did not decode DS atomic sub"); + Check(Common::ContainsStr(result.decoded_dump, "ds_min_i32"), + "new decoder did not decode DS signed min"); + Check(Common::ContainsStr(result.decoded_dump, "ds_xor_rtn_b32"), + "new decoder did not decode DS xor-return"); + Check(Common::ContainsStr(result.decoded_dump, "ds_wrxchg_rtn_b32"), + "new decoder did not decode DS write-exchange-return"); + Check(Common::ContainsStr(result.decoded_dump, "ds_add_rtn_u32"), + "new decoder did not decode DS atomic add-return"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSwapU32 v0"), + "buffer atomic swap did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v1"), + "buffer atomic add did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 v7"), + "buffer atomic sub did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 v8"), + "buffer atomic signed min did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v2"), + "buffer atomic umin did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 v9"), + "buffer atomic signed max did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v3"), + "buffer atomic umax did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v4"), + "buffer atomic and did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v5"), + "buffer atomic or did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v6"), + "buffer atomic xor did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicFMinF32 null, v0"), + "buffer atomic float min did not lower to IR without a GLC return"); + Check(Common::ContainsStr(result.ir_dump, "AtomicFMaxF32 null, v3"), + "buffer atomic float max did not lower to IR without a GLC return"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 null, v2"), + "DS no-return atomic add did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 null, v10"), + "DS no-return atomic sub did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 null, v11"), + "DS no-return signed min did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 null, v12"), + "DS no-return signed max did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 null, v13"), + "DS no-return and did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 null, v14"), + "DS no-return xor did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAddU32 v15"), + "DS add-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSubU32 v16"), + "DS sub-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMinI32 v17"), + "DS signed min-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSMaxI32 v18"), + "DS signed max-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 v19"), + "DS unsigned min-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 v20"), + "DS unsigned max-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicAndU32 v21"), + "DS and-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 v22"), + "DS or-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicXorU32 v23"), + "DS xor-return did not lower to IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicSwapU32 v24"), + "DS write-exchange-return did not lower to shared atomic swap IR"); + Check(SpirvContainsOpcode(result.spirv, 229), + "SPIR-V binary does not contain OpAtomicExchange"); + Check(SpirvContainsOpcode(result.spirv, 234), + "SPIR-V binary does not contain OpAtomicIAdd"); + Check(SpirvContainsOpcode(result.spirv, 235), + "SPIR-V binary does not contain OpAtomicISub"); + Check(SpirvContainsOpcode(result.spirv, 236), + "SPIR-V binary does not contain OpAtomicSMin"); + Check(SpirvContainsOpcode(result.spirv, 237), + "SPIR-V binary does not contain OpAtomicUMin"); + Check(SpirvContainsOpcode(result.spirv, 238), + "SPIR-V binary does not contain OpAtomicSMax"); + Check(SpirvContainsOpcode(result.spirv, 239), + "SPIR-V binary does not contain OpAtomicUMax"); + Check(SpirvContainsOpcode(result.spirv, 240), + "SPIR-V binary does not contain OpAtomicAnd"); + Check(SpirvContainsOpcode(result.spirv, 241), + "SPIR-V binary does not contain OpAtomicOr"); + Check(SpirvContainsOpcode(result.spirv, 242), + "SPIR-V binary does not contain OpAtomicXor"); + Check(SpirvContainsOpcode(result.spirv, 230), + "SPIR-V binary does not contain OpAtomicCompareExchange for float min"); + Check(SpirvContainsOpcode(result.spirv, 225), + "buffer atomic SPIR-V binary does not contain OpMemoryBarrier"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerDsReadWrite2Lowering() { - const uint32_t shader[] = { - EncodeDs0(0x0e, (3u << 8u) | 1u), - EncodeDs1Ex(0, 61, 60, 1), - EncodeDs0(0x37, (4u << 8u) | 2u), - EncodeDs1Ex(70, 0, 0, 1), - EncodeDs0(0x38, (5u << 8u) | 1u), - EncodeDs1Ex(72, 0, 0, 1), - EncodeDs0(0x77, (6u << 8u) | 2u), - EncodeDs1Ex(80, 0, 0, 1), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeDs0(0x0e, (3u << 8u) | 1u), + EncodeDs1Ex(0, 61, 60, 1), + EncodeDs0(0x37, (4u << 8u) | 2u), + EncodeDs1Ex(70, 0, 0, 1), + EncodeDs0(0x38, (5u << 8u) | 1u), + EncodeDs1Ex(72, 0, 0, 1), + EncodeDs0(0x77, (6u << 8u) | 2u), + EncodeDs1Ex(80, 0, 0, 1), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_write2_b32"), - "new decoder did not decode old-backed DS write2"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b32"), - "new decoder did not decode old-backed DS read2"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b64"), - "new decoder did not decode old-backed DS read2 b64"); - Check(Common::ContainsStr(result.decoded_dump, "offset=4"), - "DS write2 decode did not scale offset0 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset2=12"), - "DS write2 decode did not scale offset1 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset=8"), - "DS read2 decode did not scale offset0 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset2=16"), - "DS read2 decode did not scale offset1 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset=256"), - "DS read2 st64 decode did not scale offset0 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset2=1280"), - "DS read2 st64 decode did not scale offset1 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset=16"), - "DS read2 b64 decode did not scale offset0 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "offset2=48"), - "DS read2 b64 decode did not scale offset1 to bytes"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=2 bits=32"), - "DS read/write2 decode did not preserve two-dword metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), - "DS read2 b64 decode did not preserve four-dword metadata"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v60"), - "DS write2 did not lower first dword through shared LDS store IR"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v61"), - "DS write2 did not lower second dword through shared LDS store IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v70"), - "DS read2 did not lower first dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v71"), - "DS read2 did not lower second dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v72"), - "DS read2 st64 did not lower first dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v73"), - "DS read2 st64 did not lower second dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v80"), - "DS read2 b64 did not lower first dword through shared LDS load IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v83"), - "DS read2 b64 did not lower fourth dword through shared LDS load IR"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write2_b32"), + "new decoder did not decode old-backed DS write2"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b32"), + "new decoder did not decode old-backed DS read2"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read2_b64"), + "new decoder did not decode old-backed DS read2 b64"); + Check(Common::ContainsStr(result.decoded_dump, "offset=4"), + "DS write2 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=12"), + "DS write2 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset=8"), + "DS read2 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=16"), + "DS read2 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset=256"), + "DS read2 st64 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=1280"), + "DS read2 st64 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset=16"), + "DS read2 b64 decode did not scale offset0 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "offset2=48"), + "DS read2 b64 decode did not scale offset1 to bytes"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=2 bits=32"), + "DS read/write2 decode did not preserve two-dword metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "DS read2 b64 decode did not preserve four-dword metadata"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v60"), + "DS write2 did not lower first dword through shared LDS store IR"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v61"), + "DS write2 did not lower second dword through shared LDS store IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v70"), + "DS read2 did not lower first dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v71"), + "DS read2 did not lower second dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v72"), + "DS read2 st64 did not lower first dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v73"), + "DS read2 st64 did not lower second dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v80"), + "DS read2 b64 did not lower first dword through shared LDS load IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v83"), + "DS read2 b64 did not lower fourth dword through shared LDS load IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerDsSubDwordLowering() { - const uint32_t shader[] = { - EncodeDs0(0x1e, 1), EncodeDs1(0, 40, 1), // ds_write_b8 v40, v1 - EncodeDs0(0x1f, 2), EncodeDs1(0, 41, 1), // ds_write_b16 v41, v1 - EncodeDs0(0x39, 3), EncodeDs1(42, 0, 1), // ds_read_i8 v42, v1 - EncodeDs0(0x3a, 4), EncodeDs1(43, 0, 1), // ds_read_u8 v43, v1 - EncodeDs0(0x3b, 6), EncodeDs1(44, 0, 1), // ds_read_i16 v44, v1 - EncodeDs0(0x3c, 8), EncodeDs1(45, 0, 1), // ds_read_u16 v45, v1 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeDs0(0x1e, 1), EncodeDs1(0, 40, 1), // ds_write_b8 v40, v1 + EncodeDs0(0x1f, 2), EncodeDs1(0, 41, 1), // ds_write_b16 v41, v1 + EncodeDs0(0x39, 3), EncodeDs1(42, 0, 1), // ds_read_i8 v42, v1 + EncodeDs0(0x3a, 4), EncodeDs1(43, 0, 1), // ds_read_u8 v43, v1 + EncodeDs0(0x3b, 6), EncodeDs1(44, 0, 1), // ds_read_i16 v44, v1 + EncodeDs0(0x3c, 8), EncodeDs1(45, 0, 1), // ds_read_u16 v45, v1 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b8"), - "new decoder did not decode DS byte write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b16"), - "new decoder did not decode DS short write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_i8"), - "new decoder did not decode DS signed byte read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_u8"), - "new decoder did not decode DS unsigned byte read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_i16"), - "new decoder did not decode DS signed short read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_u16"), - "new decoder did not decode DS unsigned short read"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"), - "DS byte decode did not preserve byte-width metadata"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"), - "DS short decode did not preserve short-width metadata"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteByte null, v40"), - "DS byte write did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteShort null, v41"), - "DS short write did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadSbyte v42"), - "DS signed byte read did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadUbyte v43"), - "DS unsigned byte read did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadSshort v44"), - "DS signed short read did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadUshort v45"), - "DS unsigned short read did not lower to explicit IR"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 194), - "SPIR-V binary does not contain OpShiftRightLogical"); - Check(SpirvContainsOpcode(result.spirv, 195), - "SPIR-V binary does not contain OpShiftRightArithmetic"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - Check(SpirvContainsOpcode(result.spirv, 200), "SPIR-V binary does not contain OpNot"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b8"), + "new decoder did not decode DS byte write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b16"), + "new decoder did not decode DS short write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_i8"), + "new decoder did not decode DS signed byte read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_u8"), + "new decoder did not decode DS unsigned byte read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_i16"), + "new decoder did not decode DS signed short read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_u16"), + "new decoder did not decode DS unsigned short read"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=8"), + "DS byte decode did not preserve byte-width metadata"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=1 bits=16"), + "DS short decode did not preserve short-width metadata"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteByte null, v40"), + "DS byte write did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteShort null, v41"), + "DS short write did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadSbyte v42"), + "DS signed byte read did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadUbyte v43"), + "DS unsigned byte read did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadSshort v44"), + "DS signed short read did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadUshort v45"), + "DS unsigned short read did not lower to explicit IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 194), + "SPIR-V binary does not contain OpShiftRightLogical"); + Check(SpirvContainsOpcode(result.spirv, 195), + "SPIR-V binary does not contain OpShiftRightArithmetic"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + Check(SpirvContainsOpcode(result.spirv, 200), + "SPIR-V binary does not contain OpNot"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerDsWideAndAtomicLowering() { - const uint32_t shader[] = { - EncodeDs0(0x4d, 4), EncodeDs1(0, 10, 1), // ds_write_b64 v[10:11], v1 - EncodeDs0(0xde, 8), EncodeDs1(0, 12, 1), // ds_write_b96 v[12:14], v1 - EncodeDs0(0xdf, 12), EncodeDs1(0, 16, 1), // ds_write_b128 v[16:19], v1 - EncodeDs0(0x76, 16), EncodeDs1(20, 0, 1), // ds_read_b64 v[20:21], v1 - EncodeDs0(0xfe, 20), EncodeDs1(24, 0, 1), // ds_read_b96 v[24:26], v1 - EncodeDs0(0xff, 24), EncodeDs1(28, 0, 1), // ds_read_b128 v[28:31], v1 - EncodeDs0(0x07, 28), EncodeDs1(0, 32, 1), // ds_min_u32 v32, v1 - EncodeDs0(0x08, 32), EncodeDs1(0, 33, 1), // ds_max_u32 v33, v1 - EncodeDs0(0x0a, 36), EncodeDs1(0, 34, 1), // ds_or_b32 v34, v1 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeDs0(0x4d, 4), EncodeDs1(0, 10, 1), // ds_write_b64 v[10:11], v1 + EncodeDs0(0xde, 8), EncodeDs1(0, 12, 1), // ds_write_b96 v[12:14], v1 + EncodeDs0(0xdf, 12), EncodeDs1(0, 16, 1), // ds_write_b128 v[16:19], v1 + EncodeDs0(0x76, 16), EncodeDs1(20, 0, 1), // ds_read_b64 v[20:21], v1 + EncodeDs0(0xfe, 20), EncodeDs1(24, 0, 1), // ds_read_b96 v[24:26], v1 + EncodeDs0(0xff, 24), EncodeDs1(28, 0, 1), // ds_read_b128 v[28:31], v1 + EncodeDs0(0x07, 28), EncodeDs1(0, 32, 1), // ds_min_u32 v32, v1 + EncodeDs0(0x08, 32), EncodeDs1(0, 33, 1), // ds_max_u32 v33, v1 + EncodeDs0(0x0a, 36), EncodeDs1(0, 34, 1), // ds_or_b32 v34, v1 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b64"), - "new decoder did not decode DS b64 write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b96"), - "new decoder did not decode DS b96 write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_b128"), - "new decoder did not decode DS b128 write"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b64"), - "new decoder did not decode DS b64 read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b96"), - "new decoder did not decode DS b96 read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_b128"), - "new decoder did not decode DS b128 read"); - Check(Common::ContainsStr(result.decoded_dump, "ds_min_u32"), - "new decoder did not decode DS min atomic"); - Check(Common::ContainsStr(result.decoded_dump, "ds_max_u32"), - "new decoder did not decode DS max atomic"); - Check(Common::ContainsStr(result.decoded_dump, "ds_or_b32"), - "new decoder did not decode DS or atomic"); - Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), - "new decoder did not expose DS wide width metadata"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v10"), - "DS b64 write did not expand to first dword store"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v19"), - "DS b128 write did not expand to last dword store"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v20"), - "DS b64 read did not expand to first dword load"); - Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v31"), - "DS b128 read did not expand to last dword load"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 null, v32"), - "DS min atomic did not lower to shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 null, v33"), - "DS max atomic did not lower to shared atomic IR"); - Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 null, v34"), - "DS or atomic did not lower to shared atomic IR"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 237), "SPIR-V binary does not contain OpAtomicUMin"); - Check(SpirvContainsOpcode(result.spirv, 239), "SPIR-V binary does not contain OpAtomicUMax"); - Check(SpirvContainsOpcode(result.spirv, 241), "SPIR-V binary does not contain OpAtomicOr"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b64"), + "new decoder did not decode DS b64 write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b96"), + "new decoder did not decode DS b96 write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_b128"), + "new decoder did not decode DS b128 write"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b64"), + "new decoder did not decode DS b64 read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b96"), + "new decoder did not decode DS b96 read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_b128"), + "new decoder did not decode DS b128 read"); + Check(Common::ContainsStr(result.decoded_dump, "ds_min_u32"), + "new decoder did not decode DS min atomic"); + Check(Common::ContainsStr(result.decoded_dump, "ds_max_u32"), + "new decoder did not decode DS max atomic"); + Check(Common::ContainsStr(result.decoded_dump, "ds_or_b32"), + "new decoder did not decode DS or atomic"); + Check(Common::ContainsStr(result.decoded_dump, "dwords=4 bits=32"), + "new decoder did not expose DS wide width metadata"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v10"), + "DS b64 write did not expand to first dword store"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteB32 null, v19"), + "DS b128 write did not expand to last dword store"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v20"), + "DS b64 read did not expand to first dword load"); + Check(Common::ContainsStr(result.ir_dump, "DsReadB32 v31"), + "DS b128 read did not expand to last dword load"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMinU32 null, v32"), + "DS min atomic did not lower to shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicUMaxU32 null, v33"), + "DS max atomic did not lower to shared atomic IR"); + Check(Common::ContainsStr(result.ir_dump, "AtomicOrU32 null, v34"), + "DS or atomic did not lower to shared atomic IR"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 237), + "SPIR-V binary does not contain OpAtomicUMin"); + Check(SpirvContainsOpcode(result.spirv, 239), + "SPIR-V binary does not contain OpAtomicUMax"); + Check(SpirvContainsOpcode(result.spirv, 241), + "SPIR-V binary does not contain OpAtomicOr"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerDsSwizzleLowering() { - const uint32_t shader[] = { - EncodeDs0(0x35, 0x001f), - EncodeDs1(8, 0, 5), // ds_swizzle_b32 v8, v5 - EncodeDs0(0x35, 0x801b), - EncodeDs1(9, 0, 6), // ds_swizzle_b32 v9, v6 - 0xd8d4c480u, - 0x45000045u, // ds_swizzle_b32 rotate mode from boot shader - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeDs0(0x35, 0x001f), + EncodeDs1(8, 0, 5), // ds_swizzle_b32 v8, v5 + EncodeDs0(0x35, 0x801b), + EncodeDs1(9, 0, 6), // ds_swizzle_b32 v9, v6 + 0xd8d4c480u, + 0x45000045u, // ds_swizzle_b32 rotate mode from boot shader + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_swizzle_b32"), - "new decoder did not decode DS swizzle"); - Check(Common::ContainsStr(result.decoded_dump, "offset=31"), - "new decoder did not expose DS swizzle control"); - Check(Common::ContainsStr(result.decoded_dump, "offset=32795"), - "new decoder did not expose DS swizzle quad control"); - Check(Common::ContainsStr(result.decoded_dump, "offset=50304"), - "new decoder did not expose DS swizzle rotate control"); - Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v8, v5, 0x0000001f"), - "DS swizzle did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v9, v6, 0x0000801b"), - "DS quad swizzle did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v69, v69, 0x0000c480"), - "DS rotate swizzle did not lower to explicit IR"); - Check(SpirvContainsOpcode(result.spirv, 345), - "SPIR-V binary does not contain OpGroupNonUniformShuffle"); - Check(SpirvContainsOpcode(result.spirv, 128), - "SPIR-V binary does not contain OpIAdd for DS rotate swizzle"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 198), "SPIR-V binary does not contain OpBitwiseOr"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_swizzle_b32"), + "new decoder did not decode DS swizzle"); + Check(Common::ContainsStr(result.decoded_dump, "offset=31"), + "new decoder did not expose DS swizzle control"); + Check(Common::ContainsStr(result.decoded_dump, "offset=32795"), + "new decoder did not expose DS swizzle quad control"); + Check(Common::ContainsStr(result.decoded_dump, "offset=50304"), + "new decoder did not expose DS swizzle rotate control"); + Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v8, v5, 0x0000001f"), + "DS swizzle did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v9, v6, 0x0000801b"), + "DS quad swizzle did not lower to explicit IR"); + Check( + Common::ContainsStr(result.ir_dump, "DsSwizzleB32 v69, v69, 0x0000c480"), + "DS rotate swizzle did not lower to explicit IR"); + Check(SpirvContainsOpcode(result.spirv, 345), + "SPIR-V binary does not contain OpGroupNonUniformShuffle"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd for DS rotate swizzle"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 198), + "SPIR-V binary does not contain OpBitwiseOr"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerDsAddtidLowering() { - const uint32_t shader[] = { - EncodeSMovB32(124, 132), // m0 = 4 - EncodeDs0(0xb0, 8), - EncodeDs1(0, 7, 0), // ds_write_addtid_b32 v7 - EncodeDs0(0xb1, 12), - EncodeDs1(8, 0, 0), // ds_read_addtid_b32 v8 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(124, 132), // m0 = 4 + EncodeDs0(0xb0, 8), + EncodeDs1(0, 7, 0), // ds_write_addtid_b32 v7 + EncodeDs0(0xb1, 12), + EncodeDs1(8, 0, 0), // ds_read_addtid_b32 v8 + 0xbf810000u, + }; - ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); - input_info.thread_ids_num = 1; + ShaderComputeInputInfo input_info = RegressionComputeInputInfo(); + input_info.thread_ids_num = 1; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.compute_input_info = &input_info; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.compute_input_info = &input_info; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_write_addtid_b32"), - "new decoder did not decode DS write addtid"); - Check(Common::ContainsStr(result.decoded_dump, "ds_read_addtid_b32"), - "new decoder did not decode DS read addtid"); - Check(Common::ContainsStr(result.ir_dump, "DsWriteAddtidB32 null, v7, m0"), - "DS write addtid did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsReadAddtidB32 v8, m0"), - "DS read addtid did not lower to explicit IR"); - Check( - ProgramHasInput(result.program, ShaderRecompiler::IR::StageInputKind::LocalInvocationIndex), - "DS addtid did not request LocalInvocationIndex input"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 128), "SPIR-V binary does not contain OpIAdd"); - Check(SpirvContainsOpcode(result.spirv, 196), - "SPIR-V binary does not contain OpShiftLeftLogical"); - Check(SpirvContainsOpcode(result.spirv, 199), "SPIR-V binary does not contain OpBitwiseAnd"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_write_addtid_b32"), + "new decoder did not decode DS write addtid"); + Check(Common::ContainsStr(result.decoded_dump, "ds_read_addtid_b32"), + "new decoder did not decode DS read addtid"); + Check(Common::ContainsStr(result.ir_dump, "DsWriteAddtidB32 null, v7, m0"), + "DS write addtid did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsReadAddtidB32 v8, m0"), + "DS read addtid did not lower to explicit IR"); + Check(ProgramHasInput( + result.program, + ShaderRecompiler::IR::StageInputKind::LocalInvocationIndex), + "DS addtid did not request LocalInvocationIndex input"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 128), + "SPIR-V binary does not contain OpIAdd"); + Check(SpirvContainsOpcode(result.spirv, 196), + "SPIR-V binary does not contain OpShiftLeftLogical"); + Check(SpirvContainsOpcode(result.spirv, 199), + "SPIR-V binary does not contain OpBitwiseAnd"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerDsFloatMinMaxLowering() { - const uint32_t shader[] = { - EncodeDs0(0x12, 4), EncodeDs1Ex(0, 9, 7, 1), // ds_min_f32 v7, v9, v1 - EncodeDs0(0x13, 8), EncodeDs1Ex(0, 10, 8, 1), // ds_max_f32 v8, v10, v1 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeDs0(0x12, 4), EncodeDs1Ex(0, 9, 7, 1), // ds_min_f32 v7, v9, v1 + EncodeDs0(0x13, 8), EncodeDs1Ex(0, 10, 8, 1), // ds_max_f32 v8, v10, v1 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "ds_min_f32"), - "new decoder did not decode DS float min"); - Check(Common::ContainsStr(result.decoded_dump, "ds_max_f32"), - "new decoder did not decode DS float max"); - Check(Common::ContainsStr(result.ir_dump, "DsMinF32 null, v7, v1"), - "DS float min did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "DsMaxF32 null, v8, v1"), - "DS float max did not lower to explicit IR"); - Check(Common::ContainsStr(result.ir_dump, "v9"), - "DS float min did not retain DATA1 compare operand"); - Check(Common::ContainsStr(result.ir_dump, "v10"), - "DS float max did not retain DATA1 compare operand"); - Check(SpirvContainsOpcode(result.spirv, 12), "SPIR-V binary does not contain OpExtInst"); - Check(SpirvContainsOpcode(result.spirv, 61), "SPIR-V binary does not contain OpLoad"); - Check(SpirvContainsOpcode(result.spirv, 62), "SPIR-V binary does not contain OpStore"); - Check(SpirvContainsOpcode(result.spirv, 65), "SPIR-V binary does not contain OpAccessChain"); - Check(SpirvContainsOpcode(result.spirv, 124), "SPIR-V binary does not contain OpBitcast"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "ds_min_f32"), + "new decoder did not decode DS float min"); + Check(Common::ContainsStr(result.decoded_dump, "ds_max_f32"), + "new decoder did not decode DS float max"); + Check(Common::ContainsStr(result.ir_dump, "DsMinF32 null, v7, v1"), + "DS float min did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "DsMaxF32 null, v8, v1"), + "DS float max did not lower to explicit IR"); + Check(Common::ContainsStr(result.ir_dump, "v9"), + "DS float min did not retain DATA1 compare operand"); + Check(Common::ContainsStr(result.ir_dump, "v10"), + "DS float max did not retain DATA1 compare operand"); + Check(SpirvContainsOpcode(result.spirv, 12), + "SPIR-V binary does not contain OpExtInst"); + Check(SpirvContainsOpcode(result.spirv, 61), + "SPIR-V binary does not contain OpLoad"); + Check(SpirvContainsOpcode(result.spirv, 62), + "SPIR-V binary does not contain OpStore"); + Check(SpirvContainsOpcode(result.spirv, 65), + "SPIR-V binary does not contain OpAccessChain"); + Check(SpirvContainsOpcode(result.spirv, 124), + "SPIR-V binary does not contain OpBitcast"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgStraightLine() { - const uint32_t shader[] = { - EncodeSMovB32(0, 129), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 129), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "CFG:"), "CFG dump was not emitted"); - Check(Common::ContainsStr(result.ir_dump, "block_0"), "straight-line CFG block missing"); - Check(Common::ContainsStr(result.ir_dump, "successors=["), "CFG successors were not dumped"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "CFG:"), + "CFG dump was not emitted"); + Check(Common::ContainsStr(result.ir_dump, "block_0"), + "straight-line CFG block missing"); + Check(Common::ContainsStr(result.ir_dump, "successors=["), + "CFG successors were not dumped"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgIfElse() { - const uint32_t shader[] = { - EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 - EncodeSopp(0x05, 2), // s_cbranch_scc1 else - EncodeSMovB32(1, 129), // then - EncodeSopp(0x02, 1), // s_branch merge - EncodeSMovB32(1, 130), // else - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 2), // s_cbranch_scc1 else + EncodeSMovB32(1, 129), // then + EncodeSopp(0x02, 1), // s_branch merge + EncodeSMovB32(1, 130), // else + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "condition=scc1"), - "if/else branch condition missing"); - Check(SpirvContainsOpcode(result.spirv, 247), "if/else SPIR-V lacks OpSelectionMerge"); - Check(SpirvContainsOpcode(result.spirv, 250), "if/else SPIR-V lacks OpBranchConditional"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "condition=scc1"), + "if/else branch condition missing"); + Check(SpirvContainsOpcode(result.spirv, 247), + "if/else SPIR-V lacks OpSelectionMerge"); + Check(SpirvContainsOpcode(result.spirv, 250), + "if/else SPIR-V lacks OpBranchConditional"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgTerminalExitMergePS() { - const uint32_t shader[] = { - EncodeSopp(0x04, 2), // s_cbranch_scc0 end - EncodeSMovB32(0, 129), // fallthrough work - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopp(0x04, 2), // s_cbranch_scc0 end + EncodeSMovB32(0, 129), // fallthrough work + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "terminal PS branch should stay on structured path"); - Check(!Common::ContainsStr(result.ir_dump, "conditional block 0 has no structured merge"), - "known terminal PS branch shape still reports missing merge"); - Check(SpirvContainsOpcode(result.spirv, 247), - "terminal PS branch SPIR-V lacks OpSelectionMerge"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "terminal PS branch should stay on structured path"); + Check(!Common::ContainsStr(result.ir_dump, + "conditional block 0 has no structured merge"), + "known terminal PS branch shape still reports missing merge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "terminal PS branch SPIR-V lacks OpSelectionMerge"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgPostEndTargetMergePS() { - const uint32_t shader[] = { - EncodeSopp(0x04, 2), // s_cbranch_scc0 label_000c - EncodeSMovB32(0, 129), // fallthrough work - 0xbf810000u, EncodeSMovB32(1, 129), // branch target after first s_endpgm - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopp(0x04, 2), // s_cbranch_scc0 label_000c + EncodeSMovB32(0, 129), // fallthrough work + 0xbf810000u, + EncodeSMovB32(1, 129), // branch target after first s_endpgm + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - const bool ok = ShaderRecompiler::TryRecompile(shader, options, result, &error); - if (!ok) { - std::fprintf(stderr, "PostEndTargetMergePS compile error: %s\n", error.c_str()); - } - Check(ok, "post-end target PS shader failed to compile"); - Check(Common::ContainsStr(result.decoded_dump, "0x0000000c: s_mov_b32"), - "post-end branch target was not decoded"); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "post-end terminal branch should stay on structured path"); - Check(Common::ContainsStr(result.ir_dump, "pc=0x0000000c"), - "post-end branch target did not reach IR blocks"); - Check(!Common::ContainsStr(result.ir_dump, "conditional block 0 has no structured merge"), - "known post-end PS branch shape still reports missing merge"); - Check(SpirvContainsOpcode(result.spirv, 247), - "post-end terminal branch SPIR-V lacks OpSelectionMerge"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + const bool ok = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + if (!ok) { + std::fprintf(stderr, "PostEndTargetMergePS compile error: %s\n", + error.c_str()); + } + Check(ok, "post-end target PS shader failed to compile"); + Check(Common::ContainsStr(result.decoded_dump, "0x0000000c: s_mov_b32"), + "post-end branch target was not decoded"); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "post-end terminal branch should stay on structured path"); + Check(Common::ContainsStr(result.ir_dump, "pc=0x0000000c"), + "post-end branch target did not reach IR blocks"); + Check(!Common::ContainsStr(result.ir_dump, + "conditional block 0 has no structured merge"), + "known post-end PS branch shape still reports missing merge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "post-end terminal branch SPIR-V lacks OpSelectionMerge"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgLoopBreakContinue() { - const uint32_t shader[] = { - EncodeSMovB32(0, 128), // s0 = 0 - EncodeSopc(0x0a, 0, 129), // s_cmp_lt_u32 s0, 1 - EncodeSopp(0x04, 2), // break when scc == 0 - EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 - EncodeSopp(0x02, 0xfffcu), // continue/backedge - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // s0 = 0 + EncodeSopc(0x0a, 0, 129), // s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 2), // break when scc == 0 + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfffcu), // continue/backedge + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "backedge"), "loop backedge was not detected"); - Check(Common::ContainsStr(result.ir_dump, "loop_header=1"), "loop header was not marked"); - Check(Common::ContainsStr(result.ir_dump, "continue="), - "loop continue block was not identified"); - Check(SpirvContainsOpcode(result.spirv, 246), "loop SPIR-V lacks OpLoopMerge"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "backedge"), + "loop backedge was not detected"); + Check(Common::ContainsStr(result.ir_dump, "loop_header=1"), + "loop header was not marked"); + Check(Common::ContainsStr(result.ir_dump, "continue="), + "loop continue block was not identified"); + Check(SpirvContainsOpcode(result.spirv, 246), + "loop SPIR-V lacks OpLoopMerge"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() { - const uint32_t shader[] = { - EncodeSMovB32(0, 128), // preheader: s0 = 0 - EncodeSmem0(0x08, 8, 4), - 0u, // loop: s_buffer_load_dword s8, s[4:7] - EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 - EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 - EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // preheader: s0 = 0 + EncodeSmem0(0x08, 8, 4), + 0u, // loop: s_buffer_load_dword s8, s[4:7] + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 + EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop + EncodeVop1(0x01, 0, 8), + EncodeMubuf0(0x1c, 0, false), + EncodeMubuf1(0, 12, 0), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && - Common::ContainsStr(error, "unsupported GPU selection"), - "self-modifying scalar-buffer descriptor should fail explicitly"); + ShaderRecompiler::CompileResult result; + std::string error; + const bool compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(!compiled && Common::ContainsStr(error, "control-dependent phi"), + "self-modifying scalar-buffer descriptor should fail explicitly"); } void TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher() { - const uint32_t shader[] = { - EncodeSMovB32(0, 128), // preheader: s0 = 0 - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 0, 1), // loop: buffer_load_dword v0 - EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 - EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 - EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // preheader: s0 = 0 + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), // loop: + // buffer_load_dword + // v0 + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopc(0x0a, 0, 130), // s_cmp_lt_u32 s0, 2 + EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(0, 12, 0), 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && - Common::ContainsStr(error, "unsupported GPU selection"), - "self-modifying vector-buffer descriptor should fail explicitly"); + ShaderRecompiler::CompileResult result; + std::string error; + Check(!ShaderRecompiler::TryRecompile(shader, options, result, &error) && + Common::ContainsStr(error, "control-dependent phi"), + "self-modifying vector-buffer descriptor should fail explicitly"); } -void TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher() { - const uint32_t shader[] = { - EncodeSMovB32(124, 129), // m0 = one counter - EncodeDs0(0x3e), // loop: ds_append - EncodeDs1(0, 0, 0), - EncodeDs0(0x3d), // ds_consume - EncodeDs1(1, 0, 0), - EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 - EncodeSopc(0x0a, 2, 130), // s_cmp_lt_u32 s2, 2 - EncodeSopp(0x05, 0xfff9u), // s_cbranch_scc1 loop - 0xbf810000u, - }; +void TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured() { + const uint32_t shader[] = { + EncodeSMovB32(124, 129), // m0 = one counter + EncodeDs0(0x3e), // loop: ds_append + EncodeDs1(0, 0, 0), + EncodeDs0(0x3d), // ds_consume + EncodeDs1(1, 0, 0), + EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 + EncodeSopc(0x0a, 2, 130), // s_cmp_lt_u32 s2, 2 + EncodeSopp(0x05, 0xfff9u), // s_cbranch_scc1 loop + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + std::string error; + ShaderRecompiler::Decoder::Program decoded; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + const auto original_block_count = graph.blocks.size(); + const auto original_coverage = + CfgInstructionCoverage(graph, decoded.instructions.size()); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.natural_loops.size() == 1u, "DS loop was not preserved"); + Check(ShaderRecompiler::CFG::IsolateLoopHeader( + graph, graph.natural_loops.front().header, &error), + error.c_str()); + Check(graph.blocks.size() == original_block_count + 1u, + "DS loop did not add exactly one empty header"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == + original_coverage, + "DS loop canonicalization duplicated semantic instructions"); + const auto *loop_header = graph.FindBlock(graph.natural_loops.front().header); + Check(loop_header != nullptr && + loop_header->inst_begin == loop_header->inst_end, + "DS loop header still contains semantic instructions"); - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), - "DS append/consume loop header did not select dispatcher fallback"); - Check(SpirvContainsOpcode(result.spirv, 246), "DS dispatcher SPIR-V lacks OpLoopMerge"); - Check(SpirvContainsOpcode(result.spirv, 251), "DS dispatcher SPIR-V lacks OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "DS append/consume loop did not stay structured"); + Check(!result.program.dispatcher_fallback, + "DS append/consume loop unexpectedly selected dispatcher fallback"); + Check(SpirvContainsOpcode(result.spirv, 246), + "DS structured SPIR-V lacks OpLoopMerge"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgLoopHeaderDsReadStructured() { + const uint32_t shader[] = { + EncodeDs0(0x36), // loop: ds_read_b32 v0, v1 + EncodeDs1(0, 0, 1), + EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 + EncodeSopc(0x0a, 2, 130), // s_cmp_lt_u32 s2, 2 + EncodeSopp(0x05, 0xfffbu), // s_cbranch_scc1 loop + 0xbf810000u, + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured") && + !result.program.dispatcher_fallback, + "DS read loop header did not stay structured"); + Check(!SpirvContainsOpcode(result.spirv, 251), + "DS read structured SPIR-V unexpectedly contains OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() { - const uint32_t shader[] = { - EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 - EncodeSopp(0x05, 5), // outer early exit -> end - EncodeSMovB32(2, 128), // s2 = 0 - EncodeSopc(0x0a, 2, 129), // loop: s_cmp_lt_u32 s2, 1 - EncodeSopp(0x04, 2), // loop exit -> same real end block - EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 - EncodeSopp(0x02, 0xfffcu), // continue/backedge - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 5), // outer early exit -> end + EncodeSMovB32(2, 128), // s2 = 0 + EncodeSopc(0x0a, 2, 129), // loop: s_cmp_lt_u32 s2, 1 + EncodeSopp(0x04, 2), // loop exit -> same real end block + EncodeSop2(0x00, 2, 2, 129), // s_add_u32 s2, s2, 1 + EncodeSopp(0x02, 0xfffcu), // continue/backedge + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "shared outer/loop merge should stay on structured path"); - Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), - "shared outer/loop merge was not split before structurization"); - Check(SpirvContainsOpcode(result.spirv, 246), - "shared outer/loop merge SPIR-V lacks OpLoopMerge"); - Check(SpirvContainsOpcode(result.spirv, 247), - "shared outer/loop merge SPIR-V lacks OpSelectionMerge"); - Check(!SpirvContainsOpcode(result.spirv, 251), - "shared outer/loop merge unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "shared outer/loop merge should stay on structured path"); + Check( + !Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), + "shared outer/loop merge was not split before structurization"); + Check(SpirvContainsOpcode(result.spirv, 246), + "shared outer/loop merge SPIR-V lacks OpLoopMerge"); + Check(SpirvContainsOpcode(result.spirv, 247), + "shared outer/loop merge SPIR-V lacks OpSelectionMerge"); + Check(!SpirvContainsOpcode(result.spirv, 251), + "shared outer/loop merge unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() { - const uint32_t shader[] = { - EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 - EncodeSopp(0x04, 4), // loop exit -> end - EncodeSopc(0x06, 1, 1), // s_cmp_eq_u32 s1, s1 - EncodeSopp(0x04, 2), // early break -> same loop end - EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 - EncodeSopp(0x02, 0xfffau), // backedge -> loop header - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 4), // loop exit -> end + EncodeSopc(0x06, 1, 1), // s_cmp_eq_u32 s1, s1 + EncodeSopp(0x04, 2), // early break -> same loop end + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfffau), // backedge -> loop header + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "loop early-break CFG did not stay on structured path"); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, - "loop early-break SPIR-V lacks OpLoopMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "loop early-break SPIR-V unexpectedly used OpSelectionMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, - "loop early-break CFG unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "loop early-break CFG did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "loop early-break SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "loop early-break SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "loop early-break CFG unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { - const uint32_t shader[] = { - EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 - EncodeSopp(0x04, 9), // outer exit -> end - EncodeSopc(0x0a, 1, 129), // inner loop: s_cmp_lt_u32 s1, 1 - EncodeSopp(0x04, 5), // inner exit -> outer continue - EncodeSopc(0x06, 2, 2), // s_cmp_eq_u32 s2, s2 - EncodeSopp(0x05, 5), // nonlocal exit -> outer end - EncodeSMovB32(3, 129), // inner work - EncodeSop2(0x00, 1, 1, 129), // s_add_u32 s1, s1, 1 - EncodeSopp(0x02, 0xfff9u), // inner backedge - EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 - EncodeSopp(0x02, 0xfff5u), // outer backedge - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 9), // outer exit -> end + EncodeSopc(0x0a, 1, 129), // inner loop: s_cmp_lt_u32 s1, 1 + EncodeSopp(0x04, 5), // inner exit -> outer continue + EncodeSopc(0x06, 2, 2), // s_cmp_eq_u32 s2, s2 + EncodeSopp(0x05, 5), // nonlocal exit -> outer end + EncodeSMovB32(3, 129), // inner work + EncodeSop2(0x00, 1, 1, 129), // s_add_u32 s1, s1, 1 + EncodeSopp(0x02, 0xfff9u), // inner backedge + EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff5u), // outer backedge + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), - "nested-loop nonlocal exit did not select dispatcher fallback"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, - "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "nested-loop nonlocal exit did not select dispatcher fallback"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, + "nested-loop nonlocal exit dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection() { - const uint32_t shader[] = { - EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 - EncodeSopp(0x04, 6), // outer exit -> end - EncodeSopc(0x0a, 1, 129), // inner loop: s_cmp_lt_u32 s1, 1 - EncodeSopp(0x04, 2), // inner exit -> outer continue - EncodeSMovB32(2, 129), // inner work - EncodeSopp(0x02, 0xfffcu), // inner backedge - EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 - EncodeSopp(0x02, 0xfff8u), // outer backedge - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 6), // outer exit -> end + EncodeSopc(0x0a, 1, 129), // inner loop: s_cmp_lt_u32 s1, 1 + EncodeSopp(0x04, 2), // inner exit -> outer continue + EncodeSMovB32(2, 129), // inner work + EncodeSopp(0x02, 0xfffcu), // inner backedge + EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff8u), // outer backedge + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "nested local loop exit did not stay on structured path"); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2, - "nested local loop exit SPIR-V lacks both OpLoopMerge instructions"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "nested local loop exit SPIR-V unexpectedly used OpSelectionMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, - "nested local loop exit unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "nested local loop exit did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2, + "nested local loop exit SPIR-V lacks both OpLoopMerge instructions"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "nested local loop exit SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "nested local loop exit unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit() { - const uint32_t shader[] = { - EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 - EncodeSopp(0x04, 11), // outer exit -> end - EncodeSopc(0x06, 1, 1), // inner loop first exit condition - EncodeSopp(0x05, 3), // first inner exit -> tail A - EncodeSopc(0x06, 2, 2), // inner loop second exit condition - EncodeSopp(0x05, 3), // second inner exit -> tail B - EncodeSopp(0x02, 0xfffbu), // inner backedge - EncodeSMovB32(3, 129), // tail A - EncodeSopp(0x02, 2), // tail A -> outer continue - EncodeSMovB32(4, 129), // tail B - EncodeSopp(0x02, 0), // tail B -> outer continue - EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 - EncodeSopp(0x02, 0xfff3u), // outer backedge - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // outer loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 11), // outer exit -> end + EncodeSopc(0x06, 1, 1), // inner loop first exit condition + EncodeSopp(0x05, 3), // first inner exit -> tail A + EncodeSopc(0x06, 2, 2), // inner loop second exit condition + EncodeSopp(0x05, 3), // second inner exit -> tail B + EncodeSopp(0x02, 0xfffbu), // inner backedge + EncodeSMovB32(3, 129), // tail A + EncodeSopp(0x02, 2), // tail A -> outer continue + EncodeSMovB32(4, 129), // tail B + EncodeSopp(0x02, 0), // tail B -> outer continue + EncodeSop2(0x00, 0, 0, 129), // outer continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff3u), // outer backedge + 0xbf810000u, + }; - ShaderRecompiler::Decoder::Program program; - std::string error; - Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, program, &error), - error.c_str()); + ShaderRecompiler::Decoder::Program program; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, program, + &error), + error.c_str()); - ShaderRecompiler::CFG::Graph graph; - Check(ShaderRecompiler::CFG::BuildGraph(program, graph, &error), error.c_str()); - const auto original_block_count = graph.blocks.size(); - Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); - Check(graph.blocks.size() > original_block_count, - "nested loop exit tails did not create a private inner merge"); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(program, graph, &error), + error.c_str()); + const auto original_block_count = graph.blocks.size(); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.blocks.size() > original_block_count, + "nested loop exit tails did not create a private inner merge"); - const auto* outer_header = graph.FindBlockByPc(0); - const auto* inner_header = graph.FindBlockByPc(8); - Check(outer_header != nullptr && inner_header != nullptr && - outer_header->terminator.loop_header && inner_header->terminator.loop_header, - "nested loop exit-tail fixture did not retain both loop headers"); - Check(inner_header->terminator.merge_block != outer_header->terminator.continue_block, - "inner loop merge still aliases the outer continue target"); - const auto* inner_merge = graph.FindBlock(inner_header->terminator.merge_block); - Check(inner_merge != nullptr && inner_merge->inst_begin == inner_merge->inst_end && - inner_merge->terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch && - inner_merge->terminator.true_block == outer_header->terminator.continue_block, - "private inner merge does not forward to the outer continue target"); + const auto *outer_header = graph.FindBlockByPc(0); + const auto *inner_header = graph.FindBlockByPc(8); + Check(outer_header != nullptr && inner_header != nullptr && + outer_header->terminator.loop_header && + inner_header->terminator.loop_header, + "nested loop exit-tail fixture did not retain both loop headers"); + Check(inner_header->terminator.merge_block != + outer_header->terminator.continue_block, + "inner loop merge still aliases the outer continue target"); + const auto *inner_merge = + graph.FindBlock(inner_header->terminator.merge_block); + Check(inner_merge != nullptr && + inner_merge->inst_begin == inner_merge->inst_end && + inner_merge->terminator.kind == + ShaderRecompiler::CFG::TerminatorKind::Branch && + inner_merge->terminator.true_block == + outer_header->terminator.continue_block, + "private inner merge does not forward to the outer continue target"); } void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { - const uint32_t shader[] = { - EncodeSopc(0x06, 7, 7), // entry branch bypasses loop -> exit X - EncodeSopp(0x05, 5), // entry -> X - EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 - EncodeSopp(0x04, 5), // loop exit -> Y - EncodeSopc(0x06, 1, 1), // inner condition - EncodeSopp(0x05, 1), // nonmerge exit -> X, else continue - EncodeSopp(0x02, 0xfffbu), // loop backedge - EncodeSMovB32(2, 129), // X - EncodeSopp(0x02, 2), // X -> end - EncodeSMovB32(3, 129), // Y - EncodeSopp(0x02, 0), // Y -> end - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x06, 7, 7), // entry branch bypasses loop -> exit X + EncodeSopp(0x05, 5), // entry -> X + EncodeSopc(0x0a, 0, 129), // loop: s_cmp_lt_u32 s0, 1 + EncodeSopp(0x04, 5), // loop exit -> Y + EncodeSopc(0x06, 1, 1), // inner condition + EncodeSopp(0x05, 1), // nonmerge exit -> X, else continue + EncodeSopp(0x02, 0xfffbu), // loop backedge + EncodeSMovB32(2, 129), // X + EncodeSopp(0x02, 2), // X -> end + EncodeSMovB32(3, 129), // Y + EncodeSopp(0x02, 0), // Y -> end + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), - "mixed continue/nonmerge exit did not select dispatcher fallback"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, - "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "mixed continue/nonmerge exit did not select dispatcher fallback"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, + "mixed continue/nonmerge exit dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgConditionalLatchNoSelection() { - const uint32_t shader[] = { - EncodeSopp(0x02, 0), // loop header -> conditional block - EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 - EncodeSopp(0x05, 1), // loop exit -> end - EncodeSopp(0x02, 0xfffcu), // separate latch -> loop header - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopp(0x02, 0), // loop header -> conditional block + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 1), // loop exit -> end + EncodeSopp(0x02, 0xfffcu), // separate latch -> loop header + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "conditional latch did not stay on structured path"); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, - "conditional latch SPIR-V lacks OpLoopMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "conditional latch SPIR-V unexpectedly used OpSelectionMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, - "conditional latch unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "conditional latch did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "conditional latch SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "conditional latch SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "conditional latch unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() { - const uint32_t shader[] = { - EncodeSopp(0x02, 0), // loop header -> conditional latch - EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 - EncodeSopp(0x05, 0xfffdu), // direct latch backedge -> loop header - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopp(0x02, 0), // loop header -> conditional latch + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 0xfffdu), // direct latch backedge -> loop header + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "direct conditional latch did not stay on structured path"); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, - "direct conditional latch SPIR-V lacks OpLoopMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "direct conditional latch SPIR-V unexpectedly used OpSelectionMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, - "direct conditional latch unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "direct conditional latch did not stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "direct conditional latch SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "direct conditional latch SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "direct conditional latch unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { - const uint32_t shader[] = { - EncodeSMovB32(0, 128), // s0 = 0 - EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 - EncodeSopp(0x04, 9), // loop exit -> end - EncodeSopc(0x06, 1, 1), // s_cmp_eq_u32 s1, s1 - EncodeSopp(0x05, 5), // first early continue -> continue block - EncodeSopc(0x06, 2, 2), // s_cmp_eq_u32 s2, s2 - EncodeSopp(0x05, 3), // second early continue -> continue block - EncodeSopc(0x06, 3, 3), // s_cmp_eq_u32 s3, s3 - EncodeSopp(0x05, 1), // third early continue -> continue block - EncodeSMovB32(4, 129), // fallthrough work - EncodeSop2(0x00, 0, 0, 129), // continue: s_add_u32 s0, s0, 1 - EncodeSopp(0x02, 0xfff5u), // backedge -> loop header - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSMovB32(0, 128), // s0 = 0 + EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 + EncodeSopp(0x04, 9), // loop exit -> end + EncodeSopc(0x06, 1, 1), // s_cmp_eq_u32 s1, s1 + EncodeSopp(0x05, 5), // first early continue -> continue block + EncodeSopc(0x06, 2, 2), // s_cmp_eq_u32 s2, s2 + EncodeSopp(0x05, 3), // second early continue -> continue block + EncodeSopc(0x06, 3, 3), // s_cmp_eq_u32 s3, s3 + EncodeSopp(0x05, 1), // third early continue -> continue block + EncodeSMovB32(4, 129), // fallthrough work + EncodeSop2(0x00, 0, 0, 129), // continue: s_add_u32 s0, s0, 1 + EncodeSopp(0x02, 0xfff5u), // backedge -> loop header + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "loop early continues should stay on structured path"); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, - "loop early continues SPIR-V lacks OpLoopMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, - "loop early continues SPIR-V unexpectedly used OpSelectionMerge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, - "loop early continues unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "loop early continues should stay on structured path"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, + "loop early continues SPIR-V lacks OpLoopMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0, + "loop early continues SPIR-V unexpectedly used OpSelectionMerge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0, + "loop early continues unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgLoopGatewaySelection() { + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 130), // loop: s_cmp_lt_u32 s0, 2 + EncodeSopp(0x04, 8), // loop exit -> end + EncodeSopc(0x06, 1, 1), // skip-body condition + EncodeSopp(0x05, 3), // skip body -> loop-control gateway + EncodeSopc(0x06, 2, 2), // body early-break condition + EncodeSopp(0x05, 4), // early break -> end + EncodeSMovB32(3, 129), // body work + EncodeSopc(0x0a, 0, 130), // loop-control gateway + EncodeSopp(0x04, 1), // exit -> end, else latch + EncodeSopp(0x02, 0xfff6u), // latch -> loop header + 0xbf810000u, + }; + + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + const auto original_coverage = + CfgInstructionCoverage(graph, decoded.instructions.size()); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == + original_coverage, + "loop gateway structurization duplicated semantic instructions"); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured") && + !result.program.dispatcher_fallback, + "loop-control gateway selection unexpectedly selected dispatcher"); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, + "loop-control gateway SPIR-V has the wrong loop-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u, + "loop-control gateway SPIR-V has the wrong selection-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "loop-control gateway SPIR-V unexpectedly contains OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() { - const uint32_t shader[] = { - EncodeSopc(0x06, 0, 0), // loop body selection condition - EncodeSopp(0x05, 2), // select path B - EncodeSMovB32(1, 129), // path A - EncodeSopp(0x02, 1), // path A -> join - EncodeSMovB32(2, 129), // path B - EncodeSMovB32(3, 129), // join - EncodeSopc(0x06, 4, 4), // repeat condition - EncodeSopp(0x05, 0xfff8u), // repeat -> guest header - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // loop body selection condition + EncodeSopp(0x05, 2), // select path B + EncodeSMovB32(1, 129), // path A + EncodeSopp(0x02, 1), // path A -> join + EncodeSMovB32(2, 129), // path B + EncodeSMovB32(3, 129), // join + EncodeSopc(0x06, 4, 4), // repeat condition + EncodeSopp(0x05, 0xfff8u), // repeat -> guest header + 0xbf810000u, + }; - ShaderRecompiler::Decoder::Program decoded; - std::string error; - Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error), - error.c_str()); - ShaderRecompiler::CFG::Graph graph; - Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str()); - const auto original_block_count = graph.blocks.size(); - Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); - Check(graph.blocks.size() > original_block_count, - "conditional guest loop header did not create a synthetic header"); + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + const auto original_block_count = graph.blocks.size(); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.blocks.size() > original_block_count, + "conditional guest loop header did not create a synthetic header"); - uint32_t loop_headers = 0; - uint32_t selection_headers = 0; - for (const auto& block: graph.blocks) { - if (block.terminator.loop_header) { - loop_headers++; - Check(block.inst_begin == block.inst_end && - block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::Branch, - "canonical loop header is not an empty unconditional block"); - } else if (block.terminator.kind == - ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch && - block.terminator.merge_block != UINT32_MAX) { - selection_headers++; - } - } - Check(loop_headers == 1u && selection_headers == 1u, - "guest conditional was not separated from the loop header"); + uint32_t loop_headers = 0; + uint32_t selection_headers = 0; + for (const auto &block : graph.blocks) { + if (block.terminator.loop_header) { + loop_headers++; + Check(block.inst_begin == block.inst_end && + block.terminator.kind == + ShaderRecompiler::CFG::TerminatorKind::Branch, + "canonical loop header is not an empty unconditional block"); + } else if (block.terminator.kind == + ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch && + block.terminator.merge_block != UINT32_MAX) { + selection_headers++; + } + } + Check(loop_headers == 1u && selection_headers == 1u, + "guest conditional was not separated from the loop header"); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - ShaderRecompiler::CompileResult result; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, - "conditional loop-header SPIR-V has the wrong loop-merge count"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u, - "conditional loop-header SPIR-V has the wrong selection-merge count"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, - "conditional loop-header unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, + "conditional loop-header SPIR-V has the wrong loop-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u, + "conditional loop-header SPIR-V has the wrong selection-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "conditional loop-header unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgMultipleLoopLatches() { - const uint32_t shader[] = { - EncodeSopc(0x0a, 0, 129), // loop condition - EncodeSopp(0x04, 5), // loop exit -> end - EncodeSopc(0x06, 1, 1), // early repeat condition - EncodeSopp(0x05, 0xfffcu), // early repeat -> header - EncodeSMovB32(2, 129), // body - EncodeSMovB32(3, 129), // body tail - EncodeSopp(0x02, 0xfff9u), // ordinary latch -> header - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x0a, 0, 129), // loop condition + EncodeSopp(0x04, 5), // loop exit -> end + EncodeSopc(0x06, 1, 1), // early repeat condition + EncodeSopp(0x05, 0xfffcu), // early repeat -> header + EncodeSMovB32(2, 129), // body + EncodeSMovB32(3, 129), // body tail + EncodeSopp(0x02, 0xfff9u), // ordinary latch -> header + 0xbf810000u, + }; - ShaderRecompiler::Decoder::Program decoded; - std::string error; - Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error), - error.c_str()); - ShaderRecompiler::CFG::Graph graph; - Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str()); - const auto original_block_count = graph.blocks.size(); - Check(graph.back_edges.size() == 2u, "multiple-latch fixture lacks two native backedges"); - Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); - Check(graph.blocks.size() == original_block_count + 1u, - "multiple native latches did not create one synthetic continue"); - Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u, - "multiple native latches were not coalesced to one SPIR-V backedge"); - const auto& loop = graph.natural_loops.front(); - const auto* continue_block = graph.FindBlock(loop.continue_block); - Check(continue_block != nullptr && continue_block->inst_begin == continue_block->inst_end && - continue_block->predecessors.size() == 2u, - "canonical continue does not join both native latches"); + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + const auto original_block_count = graph.blocks.size(); + Check(graph.back_edges.size() == 2u, + "multiple-latch fixture lacks two native backedges"); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + Check(graph.blocks.size() == original_block_count + 1u, + "multiple native latches did not create one synthetic continue"); + Check(graph.back_edges.size() == 1u && graph.natural_loops.size() == 1u, + "multiple native latches were not coalesced to one SPIR-V backedge"); + const auto &loop = graph.natural_loops.front(); + const auto *continue_block = graph.FindBlock(loop.continue_block); + Check(continue_block != nullptr && + continue_block->inst_begin == continue_block->inst_end && + continue_block->predecessors.size() == 2u, + "canonical continue does not join both native latches"); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - ShaderRecompiler::CompileResult result; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, - "multiple-latch SPIR-V has the wrong loop-merge count"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u, - "multiple-latch SPIR-V unexpectedly used a selection merge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, - "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, + "multiple-latch SPIR-V has the wrong loop-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u, + "multiple-latch SPIR-V unexpectedly used a selection merge"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "multiple-latch SPIR-V unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() { - const uint32_t shader[] = { - EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 - EncodeSopp(0x05, 2), // first early exit -> end - EncodeSopp(0x05, 1), // second early exit -> same end merge - EncodeSMovB32(0, 129), // fallthrough work - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // s_cmp_eq_u32 s0, s0 + EncodeSopp(0x05, 2), // first early exit -> end + EncodeSopp(0x05, 1), // second early exit -> same end merge + EncodeSMovB32(0, 129), // fallthrough work + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "duplicate merge CFG did not stay on structured path"); - Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), - "duplicate merge CFG was not split before structurization"); - Check(SpirvContainsOpcode(result.spirv, 247), "duplicate merge SPIR-V lacks OpSelectionMerge"); - Check(!SpirvContainsOpcode(result.spirv, 251), - "duplicate merge CFG unexpectedly used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=structured"), + "duplicate merge CFG did not stay on structured path"); + Check( + !Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), + "duplicate merge CFG was not split before structurization"); + Check(SpirvContainsOpcode(result.spirv, 247), + "duplicate merge SPIR-V lacks OpSelectionMerge"); + Check(!SpirvContainsOpcode(result.spirv, 251), + "duplicate merge CFG unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgExecSccSharedArm() { + const uint32_t shader[] = { + EncodeSop2(0x15, 126, 4, 126), // s_andn2_b64 exec, s4, exec + EncodeSopp(0x08, 2), // execz -> shared arm + EncodeSop2(0x15, 30, 30, 126), // s_andn2_b64 s30, s30, exec + EncodeSopp(0x04, 2), // scc0 -> other arm, else shared arm + EncodeSMovB32(0, 129), // shared arm + 0xbf810000u, + EncodeSMovB32(1, 129), // other arm + 0xbf810000u, + }; + + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + const auto original_coverage = + CfgInstructionCoverage(graph, decoded.instructions.size()); + Check(std::ranges::all_of(original_coverage, + [](uint32_t uses) { return uses == 1u; }), + "shared-arm fixture already duplicated a semantic instruction"); + Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); + uint32_t route_selects = 0; + uint32_t route_sets = 0; + for (const auto &block : graph.blocks) { + route_selects += block.terminator.condition == + ShaderRecompiler::CFG::BranchCondition::GotoVariable; + route_sets += block.terminator.goto_value >= 0; + } + Check(graph.blocks.size() == 10u && route_selects == 1u && route_sets == 3u, + "shared selection arm was not routed through typed goto state"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == + original_coverage, + "EXEC/SCC shared-arm structurization changed semantic coverage"); + + ShaderRecompiler::IR::Program native_ir; + Check(ShaderRecompiler::IR::LowerProgram(decoded, graph, ShaderType::Compute, + 64, native_ir, &error), + error.c_str()); + ShaderRecompiler::IR::ValueProgram value_ir; + ShaderComputeInputInfo compute_info{}; + Check(ShaderRecompiler::Frontend::TranslateProgram( + native_ir, value_ir, nullptr, nullptr, &compute_info, &error), + error.c_str()); + uint32_t goto_sets = 0; + uint32_t goto_gets = 0; + for (const auto *block : value_ir.blocks) { + for (const auto &inst : *block) { + goto_sets += inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::SetGotoVariable; + goto_gets += inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::GetGotoVariable; + } + } + Check(goto_sets == 3u && goto_gets == 1u, + "shared-arm route was not represented by typed goto pseudo-ops"); + ShaderRecompiler::IR::RewriteToSsa(value_ir.blocks); + ShaderRecompiler::IR::RemoveIdentities(value_ir.blocks); + ShaderRecompiler::IR::EliminateDeadCode(value_ir.blocks); + for (const auto *block : value_ir.blocks) { + for (const auto &inst : *block) { + Check(inst.GetOpcode() != + ShaderRecompiler::IR::ValueOpcode::GetGotoVariable && + inst.GetOpcode() != + ShaderRecompiler::IR::ValueOpcode::SetGotoVariable, + "typed goto pseudo-op survived SSA rewriting"); + } + } + + for (const auto lane_mode : + {ShaderLaneMaskMode::NativeWave, ShaderLaneMaskMode::PerInvocation}) { + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.lane_mask_mode = lane_mode; + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(!result.program.dispatcher_fallback && + Common::ContainsStr(result.ir_dump, "mode=structured"), + "EXEC/SCC shared-arm epilogue did not stay structured"); + Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 3u, + "EXEC/SCC shared-arm SPIR-V has the wrong selection-merge count"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, + "EXEC/SCC shared-arm SPIR-V unexpectedly used dispatcher OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); + } } void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() { - const uint32_t shader[] = { - EncodeSopc(0x06, 0, 0), // block 0 - EncodeSopp(0x04, 2), // block 0 -> 2 or 1 - EncodeSopc(0x06, 1, 1), // block 1 - EncodeSopp(0x04, 6), // block 1 -> 5 or 2 - EncodeSopc(0x06, 2, 2), // block 2 - EncodeSopp(0x04, 4), // block 2 -> 5 or 3 - EncodeSopc(0x06, 3, 3), // block 3 - EncodeSopp(0x04, 2), // block 3 -> 5 or 4 - EncodeSMovB32(4, 129), // block 4 - 0xbf810000u, // block 4 -> 6 - EncodeSMovB32(5, 129), // block 5 - 0xbf810000u, // block 5 -> 6 - }; + const uint32_t shader[] = { + EncodeSopc(0x06, 0, 0), // block 0 + EncodeSopp(0x04, 2), // block 0 -> 2 or 1 + EncodeSopc(0x06, 1, 1), // block 1 + EncodeSopp(0x04, 6), // block 1 -> 5 or 2 + EncodeSopc(0x06, 2, 2), // block 2 + EncodeSopp(0x04, 4), // block 2 -> 5 or 3 + EncodeSopc(0x06, 3, 3), // block 3 + EncodeSopp(0x04, 2), // block 3 -> 5 or 4 + EncodeSMovB32(4, 129), // block 4 + 0xbf810000u, // block 4 -> 6 + EncodeSMovB32(5, 129), // block 5 + 0xbf810000u, // block 5 -> 6 + }; - ShaderRecompiler::Decoder::Program decoded; - std::string error; - Check(ShaderRecompiler::Decoder::DecodeProgram(std::span {shader}, decoded, &error), - error.c_str()); - ShaderRecompiler::CFG::Graph graph; - Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), error.c_str()); - Check(graph.blocks.size() == 7u && graph.blocks[0].successors == std::vector({1, 2}) && - graph.blocks[0].terminator.true_block == 2u && - graph.blocks[0].terminator.false_block == 1u && - graph.blocks[1].successors == std::vector({2, 5}) && - graph.blocks[1].terminator.true_block == 5u && - graph.blocks[1].terminator.false_block == 2u && - graph.blocks[2].successors == std::vector({3, 5}) && - graph.blocks[2].terminator.true_block == 5u && - graph.blocks[2].terminator.false_block == 3u && - graph.blocks[3].successors == std::vector({4, 5}) && - graph.blocks[3].terminator.true_block == 5u && - graph.blocks[3].terminator.false_block == 4u && - graph.blocks[4].successors == std::vector({6}) && - graph.blocks[5].successors == std::vector({6}), - "overlapping early-exit fixture does not match the observed shader CFG"); - Check(ShaderRecompiler::CFG::Structurize(graph, &error), error.c_str()); - std::vector reachable(graph.blocks.size()); - std::vector pending = {graph.entry_block}; - while (!pending.empty()) { - const auto block_id = pending.back(); - pending.pop_back(); - if (reachable[block_id]) { - continue; - } - reachable[block_id] = true; - pending.insert(pending.end(), graph.blocks[block_id].successors.begin(), - graph.blocks[block_id].successors.end()); - } - Check(std::all_of(reachable.begin(), reachable.end(), [](bool value) { return value; }), - "overlapping early-exit structurization left unreachable blocks"); - std::vector merges; - for (const auto& block: graph.blocks) { - if (block.terminator.kind == ShaderRecompiler::CFG::TerminatorKind::ConditionalBranch) { - Check(block.terminator.merge_block != UINT32_MAX && - std::find(merges.begin(), merges.end(), block.terminator.merge_block) == - merges.end(), - "overlapping early-exit structurization retained a shared merge"); - merges.push_back(block.terminator.merge_block); - } - } + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + Check( + graph.blocks.size() == 7u && + graph.blocks[0].successors == std::vector({1, 2}) && + graph.blocks[0].terminator.true_block == 2u && + graph.blocks[0].terminator.false_block == 1u && + graph.blocks[1].successors == std::vector({2, 5}) && + graph.blocks[1].terminator.true_block == 5u && + graph.blocks[1].terminator.false_block == 2u && + graph.blocks[2].successors == std::vector({3, 5}) && + graph.blocks[2].terminator.true_block == 5u && + graph.blocks[2].terminator.false_block == 3u && + graph.blocks[3].successors == std::vector({4, 5}) && + graph.blocks[3].terminator.true_block == 5u && + graph.blocks[3].terminator.false_block == 4u && + graph.blocks[4].successors == std::vector({6}) && + graph.blocks[5].successors == std::vector({6}), + "overlapping early-exit fixture does not match the observed shader CFG"); + const auto original_block_count = graph.blocks.size(); + const auto original_coverage = + CfgInstructionCoverage(graph, decoded.instructions.size()); + const bool structured = ShaderRecompiler::CFG::Structurize(graph, &error); + Check(!structured && Common::ContainsStr(error, "externally entered region"), + "overlapping early-exit ladder should require dispatcher without " + "cloning"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == + original_coverage, + "failed structurization duplicated semantic instructions"); + for (size_t index = original_block_count; index < graph.blocks.size(); + index++) { + Check(graph.blocks[index].inst_begin == graph.blocks[index].inst_end, + "structurization added a semantic block clone"); + } - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.dump_ir = true; - ShaderRecompiler::CompileResult result; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=structured"), - "overlapping early-exit ladder did not stay on the structured path"); - Check(!Common::ContainsStr(result.ir_dump, "duplicate structured merge block"), - "overlapping early-exit ladder retained a shared merge"); - Check(SpirvInstructionOpcodeCount(result.spirv, 247) >= 4u, - "overlapping early-exit ladder lost its selections"); - Check(SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, - "overlapping early-exit ladder used dispatcher OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(result.program.dispatcher_fallback && + Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "overlapping early-exit ladder did not select dispatcher"); + Check(Common::ContainsStr(result.ir_dump, "externally entered region"), + "overlapping early-exit ladder lost its fallback reason"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0u, + "overlapping early-exit dispatcher lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher() { + const uint32_t shader[] = { + EncodeSopp(0x02, 1), // entry -> header, skipping unreachable entry + EncodeSopp(0x02, 2), // unreachable entry -> shared selection arm + EncodeSopc(0x06, 0, 0), // header condition + EncodeSopp(0x04, 1), // header -> merge or shared arm + EncodeSMovB32(1, 129), // shared arm + 0xbf810000u, + }; + + ShaderRecompiler::Decoder::Program decoded; + std::string error; + Check(ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded, + &error), + error.c_str()); + ShaderRecompiler::CFG::Graph graph; + Check(ShaderRecompiler::CFG::BuildGraph(decoded, graph, &error), + error.c_str()); + const auto original_coverage = + CfgInstructionCoverage(graph, decoded.instructions.size()); + Check(!ShaderRecompiler::CFG::Structurize(graph, &error) && + Common::ContainsStr(error, "externally entered region block"), + "improper externally-entered selection was accepted as structured"); + Check(CfgInstructionCoverage(graph, decoded.instructions.size()) == + original_coverage, + "externally-entered selection handling duplicated semantic code"); + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + ShaderRecompiler::CompileResult result; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(result.program.dispatcher_fallback && + Common::ContainsStr(result.ir_dump, + "externally entered region block"), + "improper externally-entered selection lost its dispatcher reason"); + Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0u, + "externally-entered selection dispatcher lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerCfgIrreducibleDispatcher() { - const uint32_t shader[] = { - EncodeSopp(0x05, 2), // entry -> B, fallthrough A - EncodeSopp(0x02, 0), // A -> C - EncodeSopp(0x05, 0xfffeu), // C -> A, fallthrough B - EncodeSopp(0x02, 0xfffeu), // B -> C - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSopp(0x05, 2), // entry -> B, fallthrough A + EncodeSopp(0x02, 0), // A -> C + EncodeSopp(0x05, 0xfffeu), // C -> A, fallthrough B + EncodeSopp(0x02, 0xfffeu), // B -> C + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "irreducible CFG"), - "irreducible CFG reason was not retained"); - Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), - "irreducible CFG did not select dispatcher fallback"); - Check(SpirvContainsOpcode(result.spirv, 246), "dispatcher SPIR-V lacks OpLoopMerge"); - Check(SpirvContainsOpcode(result.spirv, 251), "dispatcher SPIR-V lacks OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "irreducible CFG"), + "irreducible CFG reason was not retained"); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "irreducible CFG did not select dispatcher fallback"); + Check(SpirvContainsOpcode(result.spirv, 246), + "dispatcher SPIR-V lacks OpLoopMerge"); + Check(SpirvContainsOpcode(result.spirv, 251), + "dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerExecMaskHelpers() { - using namespace ShaderRecompiler::Exec; + using namespace ShaderRecompiler::Exec; - State wave32; - wave32.wave_size = 32; - WriteExec(wave32, FullMask(64)); - Check(ReadExec(wave32).low == 0xffffffffu && ReadExec(wave32).high == 0u, - "wave32 EXEC high half was not masked off"); - AndExec(wave32, {0x0fu, 0xffffffffu}); - Check(ReadExec(wave32).low == 0x0fu && ReadExec(wave32).high == 0u, - "wave32 AndExec did not preserve wave size"); - InvertExec(wave32); - Check(ReadExec(wave32).low == 0xfffffff0u && ReadExec(wave32).high == 0u, - "wave32 InvertExec did not preserve wave size"); - WriteExec(wave32, {0u, 0xffffffffu}); - Check(IsExecZero(wave32), "wave32 IsExecZero used inactive high lanes"); + State wave32; + wave32.wave_size = 32; + WriteExec(wave32, FullMask(64)); + Check(ReadExec(wave32).low == 0xffffffffu && ReadExec(wave32).high == 0u, + "wave32 EXEC high half was not masked off"); + AndExec(wave32, {0x0fu, 0xffffffffu}); + Check(ReadExec(wave32).low == 0x0fu && ReadExec(wave32).high == 0u, + "wave32 AndExec did not preserve wave size"); + InvertExec(wave32); + Check(ReadExec(wave32).low == 0xfffffff0u && ReadExec(wave32).high == 0u, + "wave32 InvertExec did not preserve wave size"); + WriteExec(wave32, {0u, 0xffffffffu}); + Check(IsExecZero(wave32), "wave32 IsExecZero used inactive high lanes"); - State wave64; - wave64.wave_size = 64; - WriteExec(wave64, FullMask(64)); - Check(ReadExec(wave64).low == 0xffffffffu && ReadExec(wave64).high == 0xffffffffu, - "wave64 EXEC full mask was wrong"); - XorExec(wave64, {0xffffffffu, 0u}); - Check(ReadExec(wave64).low == 0u && ReadExec(wave64).high == 0xffffffffu, - "wave64 XorExec did not update both halves correctly"); - wave64.vcc = {0u, 1u}; - Check(!IsVccZero(wave64), "wave64 VCC high half was ignored"); + State wave64; + wave64.wave_size = 64; + WriteExec(wave64, FullMask(64)); + Check(ReadExec(wave64).low == 0xffffffffu && + ReadExec(wave64).high == 0xffffffffu, + "wave64 EXEC full mask was wrong"); + XorExec(wave64, {0xffffffffu, 0u}); + Check(ReadExec(wave64).low == 0u && ReadExec(wave64).high == 0xffffffffu, + "wave64 XorExec did not update both halves correctly"); + wave64.vcc = {0u, 1u}; + Check(!IsVccZero(wave64), "wave64 VCC high half was ignored"); } void TestComputeShaderInputWaveSize() { - const auto decode_wave_size = [](uint32_t rsrc1) { - const bool wave32 = (((rsrc1 >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) & - Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u); - return wave32 ? 32u : 64u; - }; + const auto decode_wave_size = [](uint32_t rsrc1) { + const bool wave32 = (((rsrc1 >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) & + Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u); + return wave32 ? 32u : 64u; + }; - constexpr uint32_t observed_wave32_rsrc1_a = 0x402c0146u; - constexpr uint32_t observed_wave32_rsrc1_b = 0x402c00c1u; - Check(decode_wave_size(observed_wave32_rsrc1_a) == 32u, - "COMPUTE_PGM_RSRC1 W32_EN bit did not match observed PS5 value A"); - Check(decode_wave_size(observed_wave32_rsrc1_b) == 32u, - "COMPUTE_PGM_RSRC1 W32_EN bit did not match observed PS5 value B"); + constexpr uint32_t observed_wave32_rsrc1_a = 0x402c0146u; + constexpr uint32_t observed_wave32_rsrc1_b = 0x402c00c1u; + Check(decode_wave_size(observed_wave32_rsrc1_a) == 32u, + "COMPUTE_PGM_RSRC1 W32_EN bit did not match observed PS5 value A"); + Check(decode_wave_size(observed_wave32_rsrc1_b) == 32u, + "COMPUTE_PGM_RSRC1 W32_EN bit did not match observed PS5 value B"); - constexpr uint32_t w32_en_bit = 1u << Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT; - Check(decode_wave_size(observed_wave32_rsrc1_a & ~w32_en_bit) == 64u, - "cleared COMPUTE_PGM_RSRC1 W32_EN bit did not decode as wave64"); + constexpr uint32_t w32_en_bit = 1u << Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT; + Check(decode_wave_size(observed_wave32_rsrc1_a & ~w32_en_bit) == 64u, + "cleared COMPUTE_PGM_RSRC1 W32_EN bit did not decode as wave64"); } void TestNewShaderRecompilerWave32MasksExecHighStores() { - const uint32_t shader[] = { - EncodeSop1(0x04, 126, 193), // s_mov_b64 exec, -1 - EncodeSopp(0x01), - }; + const uint32_t shader[] = { + EncodeSop1(0x04, 126, 193), // s_mov_b64 exec, -1 + EncodeSopp(0x01), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.wave_size = 32; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.wave_size = 32; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 exec_lo, -1"), - "wave32 EXEC mask regression did not decode s_mov_b64 exec, -1"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 exec_lo, -1"), + "wave32 EXEC mask regression did not decode s_mov_b64 exec, -1"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(CountSourceOccurrences(source, "OpStore %exec_hi %uint_0") >= 2u, - "wave32 EXEC high half was not clamped to zero in SPIR-V stores"); - Check(!Common::ContainsStr(source, "OpStore %exec_hi %uint_4294967295"), - "wave32 EXEC high half was stored as a full 64-lane mask"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(CountSourceOccurrences(source, "OpStore %exec_hi %uint_0") >= 2u, + "wave32 EXEC high half was not clamped to zero in SPIR-V stores"); + Check(!Common::ContainsStr(source, "OpStore %exec_hi %uint_4294967295"), + "wave32 EXEC high half was stored as a full 64-lane mask"); } void TestNewShaderRecompilerWave32VccHighScalarStores() { - const uint32_t shader[] = { - EncodeSmem0(0x09, 106, 4), - 0u, // s_buffer_load_dwordx2 vcc_lo - EncodeSopp(0x01), - }; + const uint32_t shader[] = { + EncodeSmem0(0x09, 106, 4), + 0u, // s_buffer_load_dwordx2 vcc_lo + EncodeSopp(0x01), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.wave_size = 32; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.wave_size = 32; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), - "wave32 VCC scalar regression did not lower the high VCC dword load"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "SBufferLoadDword vcc_hi"), + "wave32 VCC scalar regression did not lower the high VCC dword load"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(CountSourceOccurrences(source, "OpStore %vcc_hi") >= 2u, - "wave32 VCC high scalar load did not store to vcc_hi"); - Check(CountSourceOccurrences(source, "OpStore %vcc_hi %uint_0") == 1u, - "wave32 VCC high scalar load was clamped to zero instead of preserving data"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(CountSourceOccurrences(source, "OpStore %vcc_hi") >= 2u, + "wave32 VCC high scalar load did not store to vcc_hi"); + Check(CountSourceOccurrences(source, "OpStore %vcc_hi %uint_0") == 1u, + "wave32 VCC high scalar load was clamped to zero instead of preserving " + "data"); } void TestNewShaderRecompilerCompareMaskIsFullWaveBallot() { - const uint32_t shader[] = { - EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 - EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc - EncodeSop1(0x04, 126, 2), // s_mov_b64 exec, s[2:3] - EncodeSopp(0x01), - }; + const uint32_t shader[] = { + EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 + EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc + EncodeSop1(0x04, 126, 2), // s_mov_b64 exec, s[2:3] + EncodeSopp(0x01), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.wave_size = 32; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.wave_size = 32; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_u32"), - "compare-mask ballot regression did not decode V_CMP_LT_U32"); - Check(Common::ContainsStr(result.ir_dump, "CompareLtU32 vcc_lo"), - "compare-mask ballot regression did not lower compare to VCC"); - Check(Common::ContainsStr(result.ir_dump, "BitwiseAndU64 s2, exec_lo, vcc_lo"), - "compare-mask ballot regression did not consume VCC through scalar mask ALU"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "v_cmp_lt_u32"), + "compare-mask ballot regression did not decode V_CMP_LT_U32"); + Check(Common::ContainsStr(result.ir_dump, "CompareLtU32 vcc_lo"), + "compare-mask ballot regression did not lower compare to VCC"); + Check( + Common::ContainsStr(result.ir_dump, "BitwiseAndU64 s2, exec_lo, vcc_lo"), + "compare-mask ballot regression did not consume VCC through scalar mask " + "ALU"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(source, "OpGroupNonUniformBallot"), - "compare-mask result was not materialized with a subgroup ballot"); - Check(Common::ContainsStr(source, "OpStore %vcc_lo"), - "compare-mask ballot regression did not store VCC low mask"); - Check(Common::ContainsStr(source, "OpStore %vcc_hi %uint_0"), - "wave32 compare-mask VCC high half was not cleared"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "OpGroupNonUniformBallot"), + "compare-mask result was not materialized with a subgroup ballot"); + Check(Common::ContainsStr(source, "OpStore %vcc_lo"), + "compare-mask ballot regression did not store VCC low mask"); + Check(Common::ContainsStr(source, "OpStore %vcc_hi %uint_0"), + "wave32 compare-mask VCC high half was not cleared"); } void TestNewShaderRecompilerBufferLoadsGuardedByExec() { - const uint32_t shader[] = { - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 0, 1), // buffer_load_dword v0 - EncodeSopp(0x01), - }; + const uint32_t shader[] = { + EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), // buffer_load_dword + // v0 + EncodeMubuf0(0x1c, 0, false), EncodeMubuf1(0, 12, 0), EncodeSopp(0x01), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dword"), - "buffer load guard regression did not decode MUBUF load"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_load_dword"), + "buffer load guard regression did not decode MUBUF load"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - const auto exec_branch = Common::FindIndex(source, std::string("OpBranchConditional"), 0); - const auto array_length = Common::FindIndex(source, std::string("OpArrayLength"), 0); - const auto bounds_branch = - Common::FindIndex(source, std::string("OpBranchConditional"), array_length); - const auto element_access = - Common::FindIndex(source, std::string("OpAccessChain %_ptr_StorageBuffer_uint"), 0); - Check(exec_branch != Common::FIND_INVALID_INDEX, "buffer load SPIR-V lacks EXEC guard branch"); - Check(array_length != Common::FIND_INVALID_INDEX, - "buffer load SPIR-V lacks storage buffer array-length bounds check"); - Check(bounds_branch != Common::FIND_INVALID_INDEX, - "buffer load SPIR-V lacks storage buffer bounds branch"); - Check(element_access != Common::FIND_INVALID_INDEX, - "buffer load SPIR-V lacks storage element access"); - Check(exec_branch < array_length, "buffer load bounds check was emitted outside EXEC guard"); - Check(bounds_branch < element_access, - "buffer load storage element pointer was formed before bounds guard"); + const auto source = DisassembleSpirvBinary(result.spirv); + const auto exec_branch = + Common::FindIndex(source, std::string("OpBranchConditional"), 0); + const auto array_length = + Common::FindIndex(source, std::string("OpArrayLength"), 0); + const auto bounds_branch = Common::FindIndex( + source, std::string("OpBranchConditional"), array_length); + const auto element_access = Common::FindIndex( + source, std::string("OpAccessChain %_ptr_StorageBuffer_uint"), 0); + Check(exec_branch != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks EXEC guard branch"); + Check(array_length != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks storage buffer array-length bounds check"); + Check(bounds_branch != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks storage buffer bounds branch"); + Check(element_access != Common::FIND_INVALID_INDEX, + "buffer load SPIR-V lacks storage element access"); + Check(exec_branch < array_length, + "buffer load bounds check was emitted outside EXEC guard"); + Check(bounds_branch < element_access, + "buffer load storage element pointer was formed before bounds guard"); } void TestNewShaderRecompilerBufferAtomicsGuardedByBounds() { - const uint32_t shader[] = { - EncodeMubuf0(0x32, 0, true, true), - EncodeMubuf1(0, 0, 1), // buffer_atomic_add - EncodeSopp(0x01), - }; + const uint32_t shader[] = { + EncodeMubuf0(0x32, 0, true, true), + EncodeMubuf1(0, 0, 1), // buffer_atomic_add + EncodeSopp(0x01), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), - "buffer atomic bounds regression did not decode MUBUF atomic"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), + "buffer atomic bounds regression did not decode MUBUF atomic"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - const auto array_length = Common::FindIndex(source, std::string("OpArrayLength"), 0); - const auto bounds_branch = - Common::FindIndex(source, std::string("OpBranchConditional"), array_length); - const auto atomic = Common::FindIndex(source, std::string("OpAtomicIAdd"), 0); - const auto memory_barrier = Common::FindIndex(source, std::string("OpMemoryBarrier"), atomic); - Check(array_length != Common::FIND_INVALID_INDEX, - "buffer atomic SPIR-V lacks storage buffer array-length bounds check"); - Check(bounds_branch != Common::FIND_INVALID_INDEX, - "buffer atomic SPIR-V lacks storage buffer bounds branch"); - Check(atomic != Common::FIND_INVALID_INDEX, "buffer atomic SPIR-V lacks atomic operation"); - Check(memory_barrier != Common::FIND_INVALID_INDEX, - "buffer atomic SPIR-V lacks memory barrier after atomic operation"); - Check(bounds_branch < atomic, "buffer atomic was emitted before bounds guard"); - Check(atomic < memory_barrier, "buffer atomic memory barrier was emitted before atomic"); + const auto source = DisassembleSpirvBinary(result.spirv); + const auto array_length = + Common::FindIndex(source, std::string("OpArrayLength"), 0); + const auto bounds_branch = Common::FindIndex( + source, std::string("OpBranchConditional"), array_length); + const auto atomic = Common::FindIndex(source, std::string("OpAtomicIAdd"), 0); + const auto memory_barrier = + Common::FindIndex(source, std::string("OpMemoryBarrier"), atomic); + Check(array_length != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks storage buffer array-length bounds check"); + Check(bounds_branch != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks storage buffer bounds branch"); + Check(atomic != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks atomic operation"); + Check(memory_barrier != Common::FIND_INVALID_INDEX, + "buffer atomic SPIR-V lacks memory barrier after atomic operation"); + Check(bounds_branch < atomic, + "buffer atomic was emitted before bounds guard"); + Check(atomic < memory_barrier, + "buffer atomic memory barrier was emitted before atomic"); } void TestNewShaderRecompilerBranchConditionForms() { - struct Case { - uint32_t opcode; - const char* condition; - }; - const Case cases[] = { - {0x04, "condition=scc0"}, - {0x08, "condition=execz"}, - {0x07, "condition=vccnz"}, - }; + struct Case { + uint32_t opcode; + const char *condition; + }; + const Case cases[] = { + {0x04, "condition=scc0"}, + {0x08, "condition=execz"}, + {0x07, "condition=vccnz"}, + }; - for (const auto& c: cases) { - const uint32_t shader[] = { - EncodeSopp(c.opcode, 1), - EncodeSMovB32(0, 129), - 0xbf810000u, - }; + for (const auto &c : cases) { + const uint32_t shader[] = { + EncodeSopp(c.opcode, 1), + EncodeSMovB32(0, 129), + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, c.condition), - "branch condition was not preserved"); - Check(SpirvContainsOpcode(result.spirv, 250), - "branch condition SPIR-V lacks OpBranchConditional"); - CheckSpirvBinaryValidates(result.spirv); - if (c.opcode == 0x04) { - const auto source = DisassembleSpirvBinary(result.spirv); - Check(!Common::ContainsStr(source, "OpGroupNonUniformBallot"), - "structured SCC branch retained a host-subgroup reduction"); - } - } + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, c.condition), + "branch condition was not preserved"); + Check(SpirvContainsOpcode(result.spirv, 250), + "branch condition SPIR-V lacks OpBranchConditional"); + CheckSpirvBinaryValidates(result.spirv); + if (c.opcode == 0x04) { + const auto source = DisassembleSpirvBinary(result.spirv); + Check(!Common::ContainsStr(source, "OpGroupNonUniformBallot"), + "structured SCC branch retained a host-subgroup reduction"); + } + } } void TestNewShaderRecompilerSetpcBranch() { - const uint32_t shader[] = { - EncodeSop1(0x1f, 4, 0), // s_getpc_b64 s[4:5] - EncodeSop2(0x00, 4, 4, 140), // s_add_u32 s4, s4, 12 - EncodeSop1(0x20, 0, 4), // s_setpc_b64 s[4:5] - EncodeSMovB32(0, 129), 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSop1(0x1f, 4, 0), // s_getpc_b64 s[4:5] + EncodeSop2(0x00, 4, 4, 140), // s_add_u32 s4, s4, 12 + EncodeSop1(0x20, 0, 4), // s_setpc_b64 s[4:5] + EncodeSMovB32(0, 129), 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "s_setpc_b64"), "S_SETPC_B64 was not decoded"); - Check(Common::ContainsStr(result.ir_dump, "successors=["), - "S_SETPC_B64 did not participate in CFG"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "s_setpc_b64"), + "S_SETPC_B64 was not decoded"); + Check(Common::ContainsStr(result.ir_dump, "successors=["), + "S_SETPC_B64 did not participate in CFG"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerSetpcJumpTable() { - const uint32_t shader[] = { - EncodeSop2(0x07, 0, 0, 129), // s_min_u32 s0, s0, 1 - EncodeSop2(0x1e, 0, 0, 131), // s_lshl_b32 s0, s0, 3 - EncodeSop1(0x1f, 8, 0), // s_getpc_b64 s[8:9] - EncodeSop2(0x00, 8, 8, 255), // s_add_u32 s8, s8, literal - 0x00000038u, - EncodeSop2(0x04, 9, 9, 128), // s_addc_u32 s9, s9, 0 - EncodeSmem0(0x01, 4, 4), - 0u, // s_load_dwordx2 s[4:5], s[8:9], s0 - EncodeSopp(0x0c, 0), // s_waitcnt 0 - EncodeSop1(0x1f, 10, 0), // s_getpc_b64 s[10:11] - EncodeSop2(0x00, 10, 10, 4), // s_add_u32 s10, s10, s4 - EncodeSop2(0x04, 11, 11, 5), // s_addc_u32 s11, s11, s5 - EncodeSop1(0x20, 0, 10), // s_setpc_b64 s[10:11] - EncodeSMovB32(1, 129), // case 0 - EncodeSopp(0x02, 1), - EncodeSMovB32(2, 129), // case 1 - EncodeSopp(0x01, 0), - 0x0000000cu, - 0x00000000u, // table: target case 0 relative to pc 0x28 - 0x00000014u, - 0x00000000u, // table: target case 1 relative to pc 0x28 - }; + const uint32_t shader[] = { + EncodeSop2(0x07, 0, 0, 129), // s_min_u32 s0, s0, 1 + EncodeSop2(0x1e, 0, 0, 131), // s_lshl_b32 s0, s0, 3 + EncodeSop1(0x1f, 8, 0), // s_getpc_b64 s[8:9] + EncodeSop2(0x00, 8, 8, 255), // s_add_u32 s8, s8, literal + 0x00000038u, + EncodeSop2(0x04, 9, 9, 128), // s_addc_u32 s9, s9, 0 + EncodeSmem0(0x01, 4, 4), + 0u, // s_load_dwordx2 s[4:5], s[8:9], s0 + EncodeSopp(0x0c, 0), // s_waitcnt 0 + EncodeSop1(0x1f, 10, 0), // s_getpc_b64 s[10:11] + EncodeSop2(0x00, 10, 10, 4), // s_add_u32 s10, s10, s4 + EncodeSop2(0x04, 11, 11, 5), // s_addc_u32 s11, s11, s5 + EncodeSop1(0x20, 0, 10), // s_setpc_b64 s[10:11] + EncodeSMovB32(1, 129), // case 0 + EncodeSopp(0x02, 1), + EncodeSMovB32(2, 129), // case 1 + EncodeSopp(0x01, 0), + 0x0000000cu, + 0x00000000u, // table: target case 0 relative to pc 0x28 + 0x00000014u, + 0x00000000u, // table: target case 1 relative to pc 0x28 + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), - "S_SETPC_B64 jump table did not select dispatcher fallback"); - Check(Common::ContainsStr(result.ir_dump, "indirect_targets=2"), - "S_SETPC_B64 jump table did not retain both targets"); - Check(Common::ContainsStr(result.ir_dump, "selector_values=2"), - "S_SETPC_B64 jump table did not retain selector mapping"); - Check(!Common::ContainsStr(result.ir_dump, "SLoadDword"), - "S_SETPC_B64 jump table load was reflected as a raw scalar buffer load"); - Check(SpirvContainsOpcode(result.spirv, 251), "dispatcher SPIR-V lacks OpSwitch"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), + "S_SETPC_B64 jump table did not select dispatcher fallback"); + const auto jump = std::find_if( + result.program.values->block_info.begin(), + result.program.values->block_info.end(), [](const auto &block) { + return !block.terminator.indirect_targets.empty(); + }); + Check(jump != result.program.values->block_info.end() && + jump->terminator.indirect_targets.size() == 2, + "S_SETPC_B64 jump table did not retain both targets"); + Check(jump->terminator.indirect_selector_values.size() == 2, + "S_SETPC_B64 jump table did not retain selector mapping"); + Check( + !Common::ContainsStr(result.ir_dump, "SLoadDword"), + "S_SETPC_B64 jump table load was reflected as a raw scalar buffer load"); + Check(SpirvContainsOpcode(result.spirv, 251), + "dispatcher SPIR-V lacks OpSwitch"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerExpVertexOutputs() { - const uint32_t shader[] = { - EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 2, 3), // POS0 - EncodeExp0(0x20, 0xf), EncodeExp1(4, 5, 6, 7), // PARAM0 - EncodeExp0(0x14, 0x1), EncodeExp1(8, 0, 0, 0), // PRIM - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 2, 3), // POS0 + EncodeExp0(0x20, 0xf), EncodeExp1(4, 5, 6, 7), // PARAM0 + EncodeExp0(0x14, 0x1), EncodeExp1(8, 0, 0, 0), // PRIM + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.decoded_dump, "target=0x0c"), "POS export was not decoded"); - Check(Common::ContainsStr(result.decoded_dump, "target=0x20"), "PARAM export was not decoded"); - Check(Common::ContainsStr(result.decoded_dump, "target=0x14"), "PRIM export was not decoded"); - Check(Common::ContainsStr(result.ir_dump, "position"), "POS export did not reach IR"); - Check(Common::ContainsStr(result.ir_dump, "parameter"), "PARAM export did not reach IR"); - Check(Common::ContainsStr(result.ir_dump, "primitive"), "PRIM export did not reach IR"); - Check(SpirvContainsOpcode(result.spirv, 62), "vertex export SPIR-V lacks OpStore"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(Common::ContainsStr(result.decoded_dump, "target=0x0c"), + "POS export was not decoded"); + Check(Common::ContainsStr(result.decoded_dump, "target=0x20"), + "PARAM export was not decoded"); + Check(Common::ContainsStr(result.decoded_dump, "target=0x14"), + "PRIM export was not decoded"); + Check(Common::ContainsStr(result.ir_dump, "position"), + "POS export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "parameter"), + "PARAM export did not reach IR"); + Check(Common::ContainsStr(result.ir_dump, "primitive"), + "PRIM export did not reach IR"); + Check(SpirvContainsOpcode(result.spirv, 62), + "vertex export SPIR-V lacks OpStore"); + CheckSpirvBinaryValidates(result.spirv); +} + +void TestNewShaderRecompilerZeroInitialRegisterState() { + const uint32_t shader[] = { + EncodeVop1(0x01, 0, 0), // v_mov_b32 v0, s0 + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(!Common::ContainsStr(result.ir_dump, "UndefU32"), + "typed prologue left guest registers undefined"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(!Common::ContainsStr(source, "OpUndef"), + "zero-initialized guest registers became SPIR-V undef values"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerVertexSystemVgprs() { - using StageInputKind = ShaderRecompiler::IR::StageInputKind; + using StageInputKind = ShaderRecompiler::IR::StageInputKind; - const uint32_t shader[] = { - EncodeVop1(0x01, 0, 5 + 256), // v_mov_b32 v0, v5 - EncodeVop1(0x01, 1, 8 + 256), // v_mov_b32 v1, v8 - EncodeExp0(0x0c, 0xf), - EncodeExp1(0, 1, 2, 3), // POS0 - EncodeExp0(0x20, 0x3), - EncodeExp1(0, 1, 0, 0), // PARAM0 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeVop1(0x01, 0, 5 + 256), // v_mov_b32 v0, v5 + EncodeVop1(0x01, 1, 8 + 256), // v_mov_b32 v1, v8 + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeExp0(0x20, 0x3), + EncodeExp1(0, 1, 0, 0), // PARAM0 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(ProgramHasInput(result.program, StageInputKind::VertexIndex), - "vertex shader missing VertexIndex input"); - Check(ProgramHasInput(result.program, StageInputKind::InstanceIndex), - "vertex shader missing InstanceIndex input"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 v0, v5"), - "vertex shader did not keep v5 as a guest VGPR source"); - Check(Common::ContainsStr(result.ir_dump, "MoveU32 v1, v8"), - "vertex shader did not keep v8 as a guest VGPR source"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(ProgramHasInput(result.program, StageInputKind::VertexIndex), + "vertex shader missing VertexIndex input"); + Check(ProgramHasInput(result.program, StageInputKind::InstanceIndex), + "vertex shader missing InstanceIndex input"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v0, v5"), + "vertex shader did not keep v5 as a guest VGPR source"); + Check(Common::ContainsStr(result.ir_dump, "MoveU32 v1, v8"), + "vertex shader did not keep v8 as a guest VGPR source"); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(CountSourceOccurrences(source, "OpLoad %int %gl_VertexIndex") == 1u, - "vertex SPIR-V does not load gl_VertexIndex"); - Check(CountSourceOccurrences(source, "OpLoad %int %gl_InstanceIndex") == 1u, - "vertex SPIR-V does not load gl_InstanceIndex"); - Check(CountSourceOccurrences(source, "OpStore %v5") >= 2u, - "vertex SPIR-V does not seed guest v5 from gl_VertexIndex"); - Check(CountSourceOccurrences(source, "OpStore %v8") >= 2u, - "vertex SPIR-V does not seed guest v8 from gl_InstanceIndex"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(CountSourceOccurrences(source, "OpLoad %int %gl_VertexIndex") == 1u, + "vertex SPIR-V does not load gl_VertexIndex"); + Check(CountSourceOccurrences(source, "OpLoad %int %gl_InstanceIndex") == 1u, + "vertex SPIR-V does not load gl_InstanceIndex"); + Check(CountSourceOccurrences(source, "OpStore %v5") >= 2u, + "vertex SPIR-V does not seed guest v5 from gl_VertexIndex"); + Check(CountSourceOccurrences(source, "OpStore %v8") >= 2u, + "vertex SPIR-V does not seed guest v8 from gl_InstanceIndex"); } void TestNewShaderRecompilerVertexExportUsesLaneExecMask() { - const uint32_t shader[] = { - EncodeSop1(0x04, 126, 129), // s_mov_b64 exec, 1 - EncodeExp0(0x0c, 0xf), - EncodeExp1(0, 1, 2, 3), // POS0 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSop1(0x04, 126, 129), // s_mov_b64 exec, 1 + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(source, "gl_SubgroupInvocationID"), - "vertex export EXEC guard did not test the current lane"); - Check(Common::ContainsStr(source, "OpBranchConditional"), - "vertex export EXEC guard did not branch on the lane-active mask"); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "gl_SubgroupInvocationID"), + "vertex export EXEC guard did not test the current lane"); + Check(Common::ContainsStr(source, "OpBranchConditional"), + "vertex export EXEC guard did not branch on the lane-active mask"); - options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - ShaderRecompiler::CompileResult local_result; - Check(ShaderRecompiler::TryRecompile(shader, options, local_result, &error), error.c_str()); - Check(local_result.program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation, - "per-invocation mask mode was not preserved in the immutable program"); - CheckSpirvBinaryValidates(local_result.spirv); - const auto local_source = DisassembleSpirvBinary(local_result.spirv); - Check(!Common::ContainsStr(local_source, "OpLoad %uint %gl_SubgroupInvocationID"), - "per-invocation EXEC still depends on the native subgroup lane"); - Check(Common::ContainsStr(local_source, "OpBranchConditional"), - "per-invocation vertex export lost its EXEC guard"); + options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + ShaderRecompiler::CompileResult local_result; + Check(ShaderRecompiler::TryRecompile(shader, options, local_result, &error), + error.c_str()); + Check(local_result.program.lane_mask_mode == + ShaderLaneMaskMode::PerInvocation, + "per-invocation mask mode was not preserved in the immutable program"); + CheckSpirvBinaryValidates(local_result.spirv); + const auto local_source = DisassembleSpirvBinary(local_result.spirv); + Check(!Common::ContainsStr(local_source, + "OpLoad %uint %gl_SubgroupInvocationID"), + "per-invocation EXEC still depends on the native subgroup lane"); + Check(Common::ContainsStr(local_source, "OpBranchConditional"), + "per-invocation vertex export lost its EXEC guard"); } void TestNewShaderRecompilerPerInvocationMasks() { - const uint32_t local_shader[] = { - EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 - EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc - EncodeSop1(0x24, 4, 126), // s_and_saveexec_b64 s[4:5], exec - EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 - EncodeSop1(0x08, 126, 126), // s_not_b64 exec, exec - EncodeSop1(0x08, 126, 126), // s_not_b64 exec, exec - EncodeExp0(0x0c, 0xf), - EncodeExp1(0, 1, 2, 3), // POS0 - EncodeSopp(0x01), - }; + const uint32_t local_shader[] = { + EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 + EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc + EncodeSop1(0x24, 4, 126), // s_and_saveexec_b64 s[4:5], exec + EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 + EncodeSop1(0x08, 126, 126), // s_not_b64 exec, exec + EncodeSop1(0x08, 126, 126), // s_not_b64 exec, exec + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Vertex; - options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(local_shader, options, result, &error), error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - const auto source = DisassembleSpirvBinary(result.spirv); - Check(!Common::ContainsStr(source, "OpGroupNonUniformBallot"), - "per-invocation VCC producer still materialized a shared subgroup mask"); - Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), - "per-invocation BFM EXEC prefix still selected native subgroup lanes"); - Check(Common::ContainsStr(source, "OpLogicalAnd") && - Common::ContainsStr(source, "OpLogicalNot"), - "per-invocation wide mask ALU did not use Boolean operations"); - Check(Common::ContainsStr(source, "OpStore %vcc_lo") && - Common::ContainsStr(source, "OpStore %vcc_hi %uint_0"), - "per-invocation comparison did not store a local Boolean VCC pair"); - Check(Common::ContainsStr(result.ir_dump, "SaveexecB64 s4, exec_lo"), - "per-invocation SAVEEXEC regression did not reach IR"); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(local_shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check( + !Common::ContainsStr(source, "OpGroupNonUniformBallot"), + "per-invocation VCC producer still materialized a shared subgroup mask"); + Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), + "per-invocation BFM EXEC prefix still selected native subgroup lanes"); + Check(Common::ContainsStr(source, "OpLogicalAnd") && + Common::ContainsStr(source, "OpLogicalNot"), + "per-invocation wide mask ALU did not use Boolean operations"); + Check(Common::ContainsStr(source, "OpStore %vcc_lo") && + Common::ContainsStr(source, "OpStore %vcc_hi %uint_0"), + "per-invocation comparison did not store a local Boolean VCC pair"); + Check(Common::ContainsStr(result.ir_dump, "SaveexecB64 s4, exec_lo"), + "per-invocation SAVEEXEC regression did not reach IR"); - const uint32_t wqm_shader[] = { - EncodeSop1(0x0a, 2, 126), // s_wqm_b64 s[2:3], exec - EncodeExp0(0x0c, 0xf), - EncodeExp1(0, 1, 2, 3), // POS0 - EncodeSopp(0x01), - }; - Check(ShaderRecompiler::TryRecompile(wqm_shader, options, result, &error), error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - const auto wqm_source = DisassembleSpirvBinary(result.spirv); - Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") && - Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"), - "per-invocation scalar WQM omitted its subgroup ballot capability"); + const uint32_t wqm_shader[] = { + EncodeSop1(0x0a, 2, 126), // s_wqm_b64 s[2:3], exec + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; + Check(ShaderRecompiler::TryRecompile(wqm_shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + const auto wqm_source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") && + Common::ContainsStr(wqm_source, "OpGroupNonUniformBallot"), + "per-invocation scalar WQM omitted its subgroup ballot capability"); - const uint32_t cross_lane_shader[] = { - EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 - EncodeVop1(0x01, 0, 250), - EncodeVop1Dpp(1), // v_mov_b32 v0, v1 dpp - EncodeExp0(0x0c, 0xf), - EncodeExp1(0, 1, 2, 3), // POS0 - EncodeSopp(0x01), - }; - Check(ShaderRecompiler::TryRecompile(cross_lane_shader, options, result, &error), - error.c_str()); - CheckSpirvBinaryValidates(result.spirv); - Check(Common::ContainsStr(DisassembleSpirvBinary(result.spirv), "OpGroupNonUniformBallot"), - "per-invocation cross-lane EXEC was not reconstructed as a subgroup ballot"); + const uint32_t cross_lane_shader[] = { + EncodeSop2(0x25, 126, 132, 128), // s_bfm_b64 exec, 4, 0 + EncodeVop1(0x01, 0, 250), + EncodeVop1Dpp(1), // v_mov_b32 v0, v1 dpp + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), // POS0 + EncodeSopp(0x01), + }; + Check(ShaderRecompiler::TryRecompile(cross_lane_shader, options, result, + &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + Check(Common::ContainsStr(DisassembleSpirvBinary(result.spirv), + "OpGroupNonUniformBallot"), + "per-invocation cross-lane EXEC was not reconstructed as a subgroup " + "ballot"); +} + +void TestNewShaderRecompilerPerInvocationU64Complement() { + const uint32_t shader[] = { + EncodeVop1(0x01, 0, 0), // v_mov_b32 v0, s0 + EncodeVop1(0x01, 1, 1), // v_mov_b32 v1, s1 + EncodeVopc(0xc1, 0 + 256, 1), // v_cmp_lt_u32 vcc, v0, v1 + EncodeSop2(0x0f, 2, 126, 106), // s_and_b64 s[2:3], exec, vcc + EncodeSop1(0x08, 4, 2), // s_not_b64 s[4:5], s[2:3] + EncodeSop2(0x0f, 126, 126, 4), // s_and_b64 exec, exec, s[4:5] + EncodeExp0(0x0c, 0xf), + EncodeExp1(0, 1, 2, 3), + EncodeSopp(0x01), + }; + + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Vertex; + options.lane_mask_mode = ShaderLaneMaskMode::PerInvocation; + options.dump_ir = true; + + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + CheckSpirvBinaryValidates(result.spirv); + const auto source = DisassembleSpirvBinary(result.spirv); + Check(Common::ContainsStr(source, "OpLogicalNot"), + "per-invocation s_not_b64 did not complement the lane predicate"); + Check(!Common::ContainsStr(source, "OpNot %uint"), + "per-invocation s_not_b64 emitted raw complemented mask words"); } void TestNewShaderRecompilerExpPixelOutputs() { - const uint32_t shader[] = { - EncodeExp0(0x00, 0xf), - EncodeExp1(0, 1, 2, 3), // MRT0 - EncodeExp0(0x08, 0x1), - EncodeExp1(4, 0, 0, 0), // MRTZ depth - EncodeExp0(0x09, 0x0), - EncodeExp1(0, 0, 0, 0), // NULL - EncodeExp0(0x00, 0xf, true, true), - EncodeExp1(5, 6, 0, 0), // compressed MRT0 - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeExp0(0x00, 0xf), + EncodeExp1(0, 1, 2, 3), // MRT0 + EncodeExp0(0x08, 0x1), + EncodeExp1(4, 0, 0, 0), // MRTZ depth + EncodeExp0(0x09, 0x0), + EncodeExp1(0, 0, 0, 0), // NULL + EncodeExp0(0x00, 0xf, true, true), + EncodeExp1(5, 6, 0, 0), // compressed MRT0 + 0xbf810000u, + }; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.dump_ir = true; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.dump_ir = true; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(Common::ContainsStr(result.ir_dump, "mrt"), "MRT export did not reach IR"); - Check(Common::ContainsStr(result.ir_dump, "mrtz"), "MRTZ export did not reach IR"); - Check(Common::ContainsStr(result.ir_dump, "null"), "NULL export did not reach IR"); - Check(Common::ContainsStr(result.ir_dump, "compr=1"), "compressed MRT export did not reach IR"); - Check(SpirvContainsOpcode(result.spirv, 62), "pixel export SPIR-V lacks OpStore"); - Check(SpirvContainsExtInst(result.spirv, 62), - "compressed pixel export SPIR-V lacks GLSL.std.450 UnpackHalf2x16"); - Check(SpirvContainsOpcode(result.spirv, 81), - "compressed pixel export SPIR-V lacks OpCompositeExtract"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(SpirvContainsOpcode(result.spirv, 62), + "pixel export SPIR-V lacks OpStore"); + Check(SpirvContainsExtInst(result.spirv, 62), + "compressed pixel export SPIR-V lacks GLSL.std.450 UnpackHalf2x16"); + Check(SpirvContainsOpcode(result.spirv, 81), + "compressed pixel export SPIR-V lacks OpCompositeExtract"); + CheckSpirvBinaryValidates(result.spirv); - ShaderPixelInputInfo uint16_info; - uint16_info.target_output_mode[0] = 7; - options.pixel_input_info = &uint16_info; - ShaderRecompiler::CompileResult uint16_result; - Check(ShaderRecompiler::TryRecompile(shader, options, uint16_result, &error), error.c_str()); - const auto uint16_source = DisassembleSpirvBinary(uint16_result.spirv); - Check(Common::ContainsStr(uint16_source, "OpVariable %_ptr_Output_v4uint Output"), - "UINT16 MRT export did not use an unsigned integer output"); - Check(CountSourceOccurrences(uint16_source, "OpBitFieldUExtract") == 4u && - Common::ContainsStr(uint16_source, "%uint_0 %uint_16") && - Common::ContainsStr(uint16_source, "%uint_16 %uint_16"), - "compressed UINT16 MRT export did not extract all low/high 16-bit lanes"); - Check(!SpirvContainsExtInst(uint16_result.spirv, 62), - "compressed UINT16 MRT export was incorrectly decoded as FP16"); - CheckSpirvBinaryValidates(uint16_result.spirv); + ShaderPixelInputInfo uint16_info; + uint16_info.target_output_mode[0] = 7; + options.pixel_input_info = &uint16_info; + ShaderRecompiler::CompileResult uint16_result; + Check(ShaderRecompiler::TryRecompile(shader, options, uint16_result, &error), + error.c_str()); + const auto uint16_source = DisassembleSpirvBinary(uint16_result.spirv); + Check(Common::ContainsStr(uint16_source, + "OpVariable %_ptr_Output_v4uint Output"), + "UINT16 MRT export did not use an unsigned integer output"); + Check( + CountSourceOccurrences(uint16_source, "OpBitFieldUExtract") == 4u && + Common::ContainsStr(uint16_source, "%uint_0 %uint_16") && + Common::ContainsStr(uint16_source, "%uint_16 %uint_16"), + "compressed UINT16 MRT export did not extract all low/high 16-bit lanes"); + Check(!SpirvContainsExtInst(uint16_result.spirv, 62), + "compressed UINT16 MRT export was incorrectly decoded as FP16"); + CheckSpirvBinaryValidates(uint16_result.spirv); + + const uint32_t partial_shader[] = { + EncodeExp0(0x00, 0x7), + EncodeExp1(0, 1, 2, 3), + 0xbf810000u, + }; + options.pixel_input_info = nullptr; + ShaderRecompiler::CompileResult partial_result; + Check(ShaderRecompiler::TryRecompile(partial_shader, options, partial_result, + &error), + error.c_str()); + const auto partial_source = DisassembleSpirvBinary(partial_result.spirv); + Check(SpirvSourceHasInstructionOperand(partial_source, "OpBitcast", + "%uint_1065353216"), + "disabled float alpha export did not default to 1.0f bits"); + CheckSpirvBinaryValidates(partial_result.spirv); + + ShaderPixelInputInfo uint_info; + uint_info.target_output_mode[0] = 7; + options.pixel_input_info = &uint_info; + ShaderRecompiler::CompileResult partial_uint_result; + Check(ShaderRecompiler::TryRecompile(partial_shader, options, + partial_uint_result, &error), + error.c_str()); + const auto partial_uint_source = + DisassembleSpirvBinary(partial_uint_result.spirv); + Check(SpirvSourceHasInstructionOperand(partial_uint_source, + "OpCompositeConstruct", "%uint_1"), + "disabled integer alpha export did not default to integer one"); + CheckSpirvBinaryValidates(partial_uint_result.spirv); + + const uint32_t compressed_ba_shader[] = { + EncodeExp0(0x00, 0xc, true, true), + EncodeExp1(0, 1, 0, 0), + 0xbf810000u, + }; + ShaderRecompiler::CompileResult compressed_ba_result; + Check(ShaderRecompiler::TryRecompile(compressed_ba_shader, options, + compressed_ba_result, &error), + error.c_str()); + const auto compressed_ba_source = + DisassembleSpirvBinary(compressed_ba_result.spirv); + Check(CountSourceOccurrences(compressed_ba_source, "OpCompositeExtract") == + 1u && + CountSourceOccurrences(compressed_ba_source, + "OpBitFieldUExtract") == 2u, + "compressed UINT16 BA-only export did not read and unpack VSRC1"); + CheckSpirvBinaryValidates(compressed_ba_result.spirv); } void TestRenderTargetReverseExportMapping() { - const auto format = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16, - Prospero::ChannelType::kFloat, - Prospero::ChannelOrder::kReversed); - Check(format.format == vk::Format::eR16G16B16A16Sfloat && format.bytes_per_element == 8u, - "reverse RGBA16F render target did not retain its native host format " - "and size"); - Check(format.export_mapping == Prospero::ColorMappingAbgr && - format.export_mapping.ApplyMask(0x1u) == 0x8u && - format.export_mapping.ApplyMask(0x2u) == 0x4u && - format.export_mapping.ApplyMask(0x4u) == 0x2u && - format.export_mapping.ApplyMask(0x8u) == 0x1u && - format.export_mapping.ApplyMask(0xfu) == 0xfu, - "reverse RGBA16F render-target export or write-mask mapping is " - "incorrect"); - const auto legacy_alt = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k8_8_8_8, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kAlt); - Check(legacy_alt.format == vk::Format::eB8G8R8A8Unorm && legacy_alt.export_mapping.IsIdentity(), - "legacy BGRA render target acquired a duplicate shader export mapping"); - const auto gr32 = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k32_32, - Prospero::ChannelType::kFloat, - Prospero::ChannelOrder::kReversed); - Check(gr32.format == vk::Format::eR32G32Sfloat && gr32.bytes_per_element == 8u && - gr32.export_mapping == Prospero::ColorMappingGr, - "reverse GR32F render target did not retain its native host format, " - "size, and mapping"); + const auto format = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, + Prospero::ChannelOrder::kReversed); + Check(format.format == vk::Format::eR16G16B16A16Sfloat && + format.bytes_per_element == 8u, + "reverse RGBA16F render target did not retain its native host format " + "and size"); + Check(format.export_mapping == Prospero::ColorMappingAbgr && + format.export_mapping.ApplyMask(0x1u) == 0x8u && + format.export_mapping.ApplyMask(0x2u) == 0x4u && + format.export_mapping.ApplyMask(0x4u) == 0x2u && + format.export_mapping.ApplyMask(0x8u) == 0x1u && + format.export_mapping.ApplyMask(0xfu) == 0xfu, + "reverse RGBA16F render-target export or write-mask mapping is " + "incorrect"); + const auto legacy_alt = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k8_8_8_8, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kAlt); + Check(legacy_alt.format == vk::Format::eB8G8R8A8Unorm && + legacy_alt.export_mapping.IsIdentity(), + "legacy BGRA render target acquired a duplicate shader export mapping"); + const auto gr32 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k32_32, Prospero::ChannelType::kFloat, + Prospero::ChannelOrder::kReversed); + Check(gr32.format == vk::Format::eR32G32Sfloat && + gr32.bytes_per_element == 8u && + gr32.export_mapping == Prospero::ColorMappingGr, + "reverse GR32F render target did not retain its native host format, " + "size, and mapping"); - struct MappingCase { - Prospero::ChannelLayout layout; - Prospero::ChannelType type; - vk::Format format; - uint32_t bytes; - std::array mappings; - }; - const MappingCase mapping_cases[] = { - {Prospero::ChannelLayout::k8, - Prospero::ChannelType::kUInt, - vk::Format::eR8Uint, - 1, - {Prospero::ColorMappingRgba, Prospero::ColorMappingGr, Prospero::ColorMappingBgra, - Prospero::ColorMappingAgba}}, - {Prospero::ChannelLayout::k32_32, - Prospero::ChannelType::kFloat, - vk::Format::eR32G32Sfloat, - 8, - {Prospero::ColorMappingRgba, Prospero::ColorMappingRabg, Prospero::ColorMappingGr, - Prospero::ColorMappingArbg}}, - {Prospero::ChannelLayout::k5_6_5, - Prospero::ChannelType::kUNorm, - vk::Format::eB5G6R5UnormPack16, - 2, - {Prospero::ColorMappingRgba, Prospero::ColorMappingRgab, Prospero::ColorMappingBgra, - Prospero::ColorMappingAgbr}}, - {Prospero::ChannelLayout::k16_16_16_16, - Prospero::ChannelType::kFloat, - vk::Format::eR16G16B16A16Sfloat, - 8, - {Prospero::ColorMappingRgba, Prospero::ColorMappingBgra, Prospero::ColorMappingAbgr, - Prospero::ColorMappingArgb}}, - }; - for (const auto& mapping_case: mapping_cases) { - for (uint32_t order = 0; order < 4u; order++) { - const auto info = TextureGetRenderTargetFormat( - mapping_case.layout, mapping_case.type, - static_cast(order)); - Check(info.format == mapping_case.format && - info.bytes_per_element == mapping_case.bytes && - info.export_mapping == mapping_case.mappings[order], - "render-target channel order did not use the component-count " - "mapping"); - for (uint32_t physical = 0; physical < 4u; physical++) { - const auto logical = info.export_mapping.Map(physical); - Check(info.export_mapping.ApplyMask(1u << logical) == (1u << physical), - "logical color write mask was not mapped to the inverse physical " - "component"); - } - } - } + struct MappingCase { + Prospero::ChannelLayout layout; + Prospero::ChannelType type; + vk::Format format; + uint32_t bytes; + std::array mappings; + }; + const MappingCase mapping_cases[] = { + {Prospero::ChannelLayout::k8, + Prospero::ChannelType::kUInt, + vk::Format::eR8Uint, + 1, + {Prospero::ColorMappingRgba, Prospero::ColorMappingGr, + Prospero::ColorMappingBgra, Prospero::ColorMappingAgba}}, + {Prospero::ChannelLayout::k32_32, + Prospero::ChannelType::kFloat, + vk::Format::eR32G32Sfloat, + 8, + {Prospero::ColorMappingRgba, Prospero::ColorMappingRabg, + Prospero::ColorMappingGr, Prospero::ColorMappingArbg}}, + {Prospero::ChannelLayout::k5_6_5, + Prospero::ChannelType::kUNorm, + vk::Format::eB5G6R5UnormPack16, + 2, + {Prospero::ColorMappingRgba, Prospero::ColorMappingRgab, + Prospero::ColorMappingBgra, Prospero::ColorMappingAgbr}}, + {Prospero::ChannelLayout::k16_16_16_16, + Prospero::ChannelType::kFloat, + vk::Format::eR16G16B16A16Sfloat, + 8, + {Prospero::ColorMappingRgba, Prospero::ColorMappingBgra, + Prospero::ColorMappingAbgr, Prospero::ColorMappingArgb}}, + }; + for (const auto &mapping_case : mapping_cases) { + for (uint32_t order = 0; order < 4u; order++) { + const auto info = TextureGetRenderTargetFormat( + mapping_case.layout, mapping_case.type, + static_cast(order)); + Check(info.format == mapping_case.format && + info.bytes_per_element == mapping_case.bytes && + info.export_mapping == mapping_case.mappings[order], + "render-target channel order did not use the component-count " + "mapping"); + for (uint32_t physical = 0; physical < 4u; physical++) { + const auto logical = info.export_mapping.Map(physical); + Check(info.export_mapping.ApplyMask(1u << logical) == (1u << physical), + "logical color write mask was not mapped to the inverse physical " + "component"); + } + } + } - const auto alt_1010102 = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k10_10_10_2, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kAlt); - Check(alt_1010102.format == vk::Format::eA2R10G10B10UnormPack32 && - alt_1010102.export_mapping.IsIdentity(), - "native alternate 10:10:10:2 render target acquired a shader mapping"); - const auto reversed_4444 = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k4_4_4_4, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kReversed); - Check(reversed_4444.format == vk::Format::eR4G4B4A4UnormPack16 && - reversed_4444.export_mapping.IsIdentity(), - "reversed 4:4:4:4 render target did not compose storage and channel " - "order"); - const auto standard_4444 = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k4_4_4_4, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kStandard); - Check(standard_4444.format == vk::Format::eR4G4B4A4UnormPack16 && - standard_4444.export_mapping == Prospero::ColorMappingAbgr, - "standard 4:4:4:4 render target did not compensate host packed-bit " - "order"); - const auto standard_5551 = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k5_5_5_1, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kStandard); - Check(standard_5551.format == vk::Format::eA1R5G5B5UnormPack16 && - standard_5551.export_mapping == Prospero::ColorMappingBgra, - "5:5:5:1 render target did not preserve its high one-bit component"); - const auto standard_1555 = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k1_5_5_5, - Prospero::ChannelType::kUNorm, - Prospero::ChannelOrder::kStandard); - Check(standard_1555.format == vk::Format::eR5G5B5A1UnormPack16 && - standard_1555.export_mapping == Prospero::ColorMappingAbgr, - "1:5:5:5 render target did not preserve its low one-bit component"); + const auto alt_1010102 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k10_10_10_2, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kAlt); + Check(alt_1010102.format == vk::Format::eA2R10G10B10UnormPack32 && + alt_1010102.export_mapping.IsIdentity(), + "native alternate 10:10:10:2 render target acquired a shader mapping"); + const auto reversed_4444 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k4_4_4_4, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kReversed); + Check(reversed_4444.format == vk::Format::eR4G4B4A4UnormPack16 && + reversed_4444.export_mapping.IsIdentity(), + "reversed 4:4:4:4 render target did not compose storage and channel " + "order"); + const auto standard_4444 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k4_4_4_4, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kStandard); + Check(standard_4444.format == vk::Format::eR4G4B4A4UnormPack16 && + standard_4444.export_mapping == Prospero::ColorMappingAbgr, + "standard 4:4:4:4 render target did not compensate host packed-bit " + "order"); + const auto standard_5551 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k5_5_5_1, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kStandard); + Check(standard_5551.format == vk::Format::eA1R5G5B5UnormPack16 && + standard_5551.export_mapping == Prospero::ColorMappingBgra, + "5:5:5:1 render target did not preserve its high one-bit component"); + const auto standard_1555 = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k1_5_5_5, Prospero::ChannelType::kUNorm, + Prospero::ChannelOrder::kStandard); + Check(standard_1555.format == vk::Format::eR5G5B5A1UnormPack16 && + standard_1555.export_mapping == Prospero::ColorMappingAbgr, + "1:5:5:5 render target did not preserve its low one-bit component"); - const auto argb = - TextureGetRenderTargetFormat(Prospero::ChannelLayout::k16_16_16_16, - Prospero::ChannelType::kFloat, - Prospero::ChannelOrder::kAltReversed); - Check(argb.export_mapping == Prospero::ColorMappingArgb && - argb.export_mapping.ApplyMask(0x1u) == 0x2u && - argb.export_mapping.ApplyMask(0x2u) == 0x4u && - argb.export_mapping.ApplyMask(0x4u) == 0x8u && - argb.export_mapping.ApplyMask(0x8u) == 0x1u, - "alternate-reversed render-target mapping did not invert its " - "non-involutive cycle"); + const auto argb = TextureGetRenderTargetFormat( + Prospero::ChannelLayout::k16_16_16_16, Prospero::ChannelType::kFloat, + Prospero::ChannelOrder::kAltReversed); + Check(argb.export_mapping == Prospero::ColorMappingArgb && + argb.export_mapping.ApplyMask(0x1u) == 0x2u && + argb.export_mapping.ApplyMask(0x2u) == 0x4u && + argb.export_mapping.ApplyMask(0x4u) == 0x8u && + argb.export_mapping.ApplyMask(0x8u) == 0x1u, + "alternate-reversed render-target mapping did not invert its " + "non-involutive cycle"); - const uint32_t shader[] = { - EncodeExp0(0x00, 0xf), - EncodeExp1(0, 1, 2, 3), // MRT0 - 0xbf810000u, - }; - ShaderPixelInputInfo identity_info; - ShaderPixelInputInfo reversed_info; - reversed_info.target_export_mapping[0] = format.export_mapping; + const uint32_t shader[] = { + EncodeExp0(0x00, 0xf), + EncodeExp1(0, 1, 2, 3), // MRT0 + 0xbf810000u, + }; + ShaderPixelInputInfo identity_info; + ShaderPixelInputInfo reversed_info; + reversed_info.target_export_mapping[0] = format.export_mapping; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.pixel_input_info = &identity_info; - ShaderRecompiler::CompileResult identity_result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, identity_result, &error), error.c_str()); - Check(SpirvInstructionOpcodeCount(identity_result.spirv, 79u) == 0u, - "identity MRT export unexpectedly added a component shuffle"); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &identity_info; + ShaderRecompiler::CompileResult identity_result; + std::string error; + Check( + ShaderRecompiler::TryRecompile(shader, options, identity_result, &error), + error.c_str()); + Check(SpirvInstructionOpcodeCount(identity_result.spirv, 79u) == 0u, + "identity MRT export unexpectedly added a component shuffle"); - options.pixel_input_info = &reversed_info; - ShaderRecompiler::CompileResult reversed_result; - Check(ShaderRecompiler::TryRecompile(shader, options, reversed_result, &error), error.c_str()); - Check(SpirvContainsVectorShuffle(reversed_result.spirv, {3u, 2u, 1u, 0u}), - "reverse MRT export did not emit a WZYX component shuffle"); - CheckSpirvBinaryValidates(reversed_result.spirv); + options.pixel_input_info = &reversed_info; + ShaderRecompiler::CompileResult reversed_result; + Check( + ShaderRecompiler::TryRecompile(shader, options, reversed_result, &error), + error.c_str()); + Check(SpirvContainsVectorShuffle(reversed_result.spirv, {3u, 2u, 1u, 0u}), + "reverse MRT export did not emit a WZYX component shuffle"); + CheckSpirvBinaryValidates(reversed_result.spirv); - reversed_info.target_export_mapping[0] = gr32.export_mapping; - ShaderRecompiler::CompileResult gr32_result; - Check(ShaderRecompiler::TryRecompile(shader, options, gr32_result, &error), error.c_str()); - Check(SpirvContainsVectorShuffle(gr32_result.spirv, {1u, 0u, 2u, 3u}), - "reverse GR32F MRT export did not emit a YXZW component shuffle"); - CheckSpirvBinaryValidates(gr32_result.spirv); + reversed_info.target_export_mapping[0] = gr32.export_mapping; + ShaderRecompiler::CompileResult gr32_result; + Check(ShaderRecompiler::TryRecompile(shader, options, gr32_result, &error), + error.c_str()); + Check(SpirvContainsVectorShuffle(gr32_result.spirv, {1u, 0u, 2u, 3u}), + "reverse GR32F MRT export did not emit a YXZW component shuffle"); + CheckSpirvBinaryValidates(gr32_result.spirv); - reversed_info.target_export_mapping[0] = argb.export_mapping; - ShaderRecompiler::CompileResult argb_result; - Check(ShaderRecompiler::TryRecompile(shader, options, argb_result, &error), error.c_str()); - Check(SpirvContainsVectorShuffle(argb_result.spirv, {3u, 0u, 1u, 2u}), - "alternate-reversed MRT export did not emit an ARGB component shuffle"); - CheckSpirvBinaryValidates(argb_result.spirv); - reversed_info.target_export_mapping[0] = gr32.export_mapping; + reversed_info.target_export_mapping[0] = argb.export_mapping; + ShaderRecompiler::CompileResult argb_result; + Check(ShaderRecompiler::TryRecompile(shader, options, argb_result, &error), + error.c_str()); + Check(SpirvContainsVectorShuffle(argb_result.spirv, {3u, 0u, 1u, 2u}), + "alternate-reversed MRT export did not emit an ARGB component shuffle"); + CheckSpirvBinaryValidates(argb_result.spirv); + reversed_info.target_export_mapping[0] = gr32.export_mapping; - HW::PixelShaderInfo regs {}; - Check(ShaderGetIdPS(regs, identity_info, false) != ShaderGetIdPS(regs, reversed_info, false), - "pixel shader cache identity omitted the render-target export mapping"); + HW::PixelShaderInfo regs{}; + Check(ShaderGetIdPS(regs, identity_info, false) != + ShaderGetIdPS(regs, reversed_info, false), + "pixel shader cache identity omitted the render-target export mapping"); - regs.ps_regs.data_addr = reinterpret_cast(shader); - regs.ps_regs.chksum = 0xf16ab6f000000001ull; - ShaderMappedData mapped {}; - mapped.code_size_bytes = sizeof(shader); - ShaderMapUserData(regs.ps_regs.data_addr, mapped); - HW::ShaderRegisters sh {}; - ShaderVertexInputInfo vs_info {}; - vs_info.stage.program = std::make_shared(); - std::array mappings {}; - mappings[0] = gr32.export_mapping; - ShaderPixelInputInfo compiled_info {}; - std::span compiled_spirv; - Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, mappings, - compiled_info, compiled_spirv) && - compiled_info.target_export_mapping[0].IsIdentity(), - "inactive reverse MRT mapping was not normalized out of the shader " - "cache key"); - sh.target_output_mode[0] = 4; - Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, mappings, - compiled_info, compiled_spirv) && - compiled_info.target_export_mapping[0] == gr32.export_mapping, - "active reverse MRT mapping was lost before shader specialization"); + regs.ps_regs.data_addr = reinterpret_cast(shader); + regs.ps_regs.chksum = 0xf16ab6f000000001ull; + ShaderMappedData mapped{}; + mapped.code_size_bytes = sizeof(shader); + ShaderMapUserData(regs.ps_regs.data_addr, mapped); + HW::ShaderRegisters sh{}; + ShaderVertexInputInfo vs_info{}; + vs_info.stage.program = std::make_shared(); + std::array mappings{}; + mappings[0] = gr32.export_mapping; + ShaderPixelInputInfo compiled_info{}; + std::span compiled_spirv; + Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, + mappings, compiled_info, compiled_spirv) && + compiled_info.target_export_mapping[0].IsIdentity(), + "inactive reverse MRT mapping was not normalized out of the shader " + "cache key"); + sh.target_output_mode[0] = 4; + Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, + mappings, compiled_info, compiled_spirv) && + compiled_info.target_export_mapping[0] == gr32.export_mapping, + "active reverse MRT mapping was lost before shader specialization"); } void TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled() { - constexpr uint32_t ExecutionModeEarlyFragmentTests = 9; + constexpr uint32_t ExecutionModeEarlyFragmentTests = 9; - const uint32_t shader[] = { - EncodeExp0(0x00, 0xf, true, false, true), - EncodeExp1(0, 1, 2, 3), - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeExp0(0x00, 0xf, true, false, true), + EncodeExp1(0, 1, 2, 3), + 0xbf810000u, + }; - ShaderPixelInputInfo ps_info; - ps_info.ps_early_z = true; - ps_info.ps_pixel_kill_enable = true; + ShaderPixelInputInfo ps_info; + ps_info.ps_early_z = true; + ps_info.ps_pixel_kill_enable = true; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.pixel_input_info = &ps_info; + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.pixel_input_info = &ps_info; - ShaderRecompiler::CompileResult result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), error.c_str()); - Check(SpirvContainsOpcode(result.spirv, 252), - "pixel valid-mask export should still lower to OpKill"); - Check(!SpirvContainsExecutionMode(result.spirv, ExecutionModeEarlyFragmentTests), - "pixel shaders that may kill fragments must not request EarlyFragmentTests"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, options, result, &error), + error.c_str()); + Check(SpirvInstructionOpcodeCount(result.spirv, 252) != 0, + "pixel valid-mask export should still lower to OpKill"); + Check(!SpirvContainsExecutionMode(result.spirv, + ExecutionModeEarlyFragmentTests), + "pixel shaders that may kill fragments must not request " + "EarlyFragmentTests"); + CheckSpirvBinaryValidates(result.spirv); + + const uint32_t ordinary_shader[] = { + EncodeExp0(0x00, 0xf), + EncodeExp1(0, 1, 2, 3), + 0xbf810000u, + }; + ps_info.ps_pixel_kill_enable = false; + ShaderRecompiler::CompileResult ordinary_result; + Check(ShaderRecompiler::TryRecompile(ordinary_shader, options, + ordinary_result, &error), + error.c_str()); + const auto ordinary_source = DisassembleSpirvBinary(ordinary_result.spirv); + Check(SpirvInstructionOpcodeCount(ordinary_result.spirv, 252) == 0, + "ordinary pixel shader unexpectedly contains OpKill"); + Check(!Common::ContainsStr(ordinary_source, "pixel_valid_mask_active"), + "ordinary pixel shader allocated pixel-valid state"); + Check(SpirvContainsExecutionMode(ordinary_result.spirv, + ExecutionModeEarlyFragmentTests), + "ordinary early-Z pixel shader lost EarlyFragmentTests"); + CheckSpirvBinaryValidates(ordinary_result.spirv); } void TestNewShaderRecompilerNativeBindingPlan() { - using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; + using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind; - const uint32_t shader[] = { - EncodeSmem0(0x08, 0, 4), - 0u, // s_buffer_load_dword s0 - EncodeMubuf0(0x1c, 8), - EncodeMubuf1(0, 1, 1), // buffer_store_dword - EncodeMimg0(0x20, 0xf), - EncodeMimg1(8, 2, 3, 1), // image_sample - EncodeMimg0(0x08, 0xf), - EncodeMimg1(12, 5, 0, 1), // image_store - EncodeMimg0(0x11, 0x1), - EncodeMimg1(16, 6, 0, 1), // image_atomic_add - 0xbf810000u, - }; + const uint32_t shader[] = { + EncodeSmem0(0x08, 0, 4), + 0u, // s_buffer_load_dword s0 + EncodeVop1(0x01, 0, 0), + EncodeMubuf0(0x1c, 8), + EncodeMubuf1(0, 1, 1), // buffer_store_dword + EncodeMimg0(0x20, 0xf), + EncodeMimg1(8, 2, 3, 1), // image_sample + EncodeMubuf0(0x1c, 12), + EncodeMubuf1(8, 1, 1), // keep sampled value live + EncodeMimg0(0x08, 0xf), + EncodeMimg1(12, 5, 0, 1), // image_store + EncodeMimg0(0x11, 0x1), + EncodeMimg1(16, 6, 0, 1), // image_atomic_add + 0xbf810000u, + }; - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.dump_ir = true; - options.user_data = user_data.data(); + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.dump_ir = true; + options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - std::string error; - const auto compiled = ShaderRecompiler::TryRecompile(shader, options, result, &error); - Check(compiled, error.c_str()); - const auto* buffers = - ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Buffers); - const auto* sampled = - ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Sampled2D); - const auto* storage = - ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Storage2D); - const auto* uint_storage = - ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::StorageUint2D); - const auto* samplers = - ShaderRecompiler::IR::FindBinding(result.program.bindings, BindingKind::Samplers); - Check(buffers != nullptr && buffers->resources.size() == 2, - "native binding plan did not allocate scalar/vector buffers"); - Check(sampled != nullptr && sampled->resources.size() == 1, - "native binding plan did not allocate the sampled image"); - Check(storage != nullptr && storage->resources.size() == 1, - "native binding plan did not allocate the float storage image"); - Check(uint_storage != nullptr && uint_storage->resources.size() == 1, - "native binding plan did not allocate the uint storage image"); - Check(samplers != nullptr && samplers->resources.size() == 1, - "native binding plan did not allocate the sampler"); - Check(SpirvContainsOpcode(result.spirv, 86), - "SPIR-V binary does not combine separate image/sampler descriptors"); - Check(SpirvHasDecorationValue(result.spirv, 33u, buffers->binding), - "SPIR-V lacks storage-buffer Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 33u, sampled->binding), - "SPIR-V lacks sampled-image Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 33u, samplers->binding), - "SPIR-V lacks sampler Binding decoration"); - Check(SpirvHasDecorationValue(result.spirv, 34u, result.program.bindings.descriptor_set), - "SPIR-V lacks DescriptorSet decoration"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + const auto compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(compiled, error.c_str()); + const auto *buffers = ShaderRecompiler::IR::FindBinding( + result.program.bindings, BindingKind::Buffers); + const auto *sampled = ShaderRecompiler::IR::FindBinding( + result.program.bindings, BindingKind::Sampled2D); + const auto *storage = ShaderRecompiler::IR::FindBinding( + result.program.bindings, BindingKind::Storage2D); + const auto *uint_storage = ShaderRecompiler::IR::FindBinding( + result.program.bindings, BindingKind::StorageUint2D); + const auto *samplers = ShaderRecompiler::IR::FindBinding( + result.program.bindings, BindingKind::Samplers); + Check(buffers != nullptr && buffers->resources.size() == 2, + "native binding plan did not allocate scalar/vector buffers"); + Check(sampled != nullptr && sampled->resources.size() == 1, + "native binding plan did not allocate the sampled image"); + Check(storage != nullptr && storage->resources.size() == 1, + "native binding plan did not allocate the float storage image"); + Check(uint_storage != nullptr && uint_storage->resources.size() == 1, + "native binding plan did not allocate the uint storage image"); + Check(samplers != nullptr && samplers->resources.size() == 1, + "native binding plan did not allocate the sampler"); + Check(SpirvContainsOpcode(result.spirv, 86), + "SPIR-V binary does not combine separate image/sampler descriptors"); + Check(SpirvHasDecorationValue(result.spirv, 33u, buffers->binding), + "SPIR-V lacks storage-buffer Binding decoration"); + Check(SpirvHasDecorationValue(result.spirv, 33u, sampled->binding), + "SPIR-V lacks sampled-image Binding decoration"); + Check(SpirvHasDecorationValue(result.spirv, 33u, samplers->binding), + "SPIR-V lacks sampler Binding decoration"); + Check(SpirvHasDecorationValue(result.spirv, 34u, + result.program.bindings.descriptor_set), + "SPIR-V lacks DescriptorSet decoration"); + CheckSpirvBinaryValidates(result.spirv); - auto malformed = result.program; - auto malformed_buffers = - std::find_if(malformed.bindings.descriptors.begin(), malformed.bindings.descriptors.end(), - [](const auto& binding) { return binding.kind == BindingKind::Buffers; }); - Check(malformed_buffers != malformed.bindings.descriptors.end(), - "native validation fixture lacks a buffer group"); - malformed_buffers->resources.clear(); - std::vector rejected_spirv = {0xdeadbeefu}; - const auto rejected_before = rejected_spirv; - std::string rejected_error; - Check(!ShaderRecompiler::Spirv::EmitProgram(malformed, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("topology") != std::string::npos, - "malformed native binding plan was not rejected transactionally"); - const auto FindBufferInstruction = [](auto* program) { - for (auto& block: program->blocks) { - const auto found = std::find_if( - block.instructions.begin(), block.instructions.end(), [](const auto& inst) { - return inst.memory.kind == ShaderRecompiler::IR::ResourceKind::Buffer; - }); - if (found != block.instructions.end()) { - return &*found; - } - } - return static_cast(nullptr); - }; + auto malformed = result.program; + auto malformed_buffers = std::find_if( + malformed.bindings.descriptors.begin(), + malformed.bindings.descriptors.end(), + [](const auto &binding) { return binding.kind == BindingKind::Buffers; }); + Check(malformed_buffers != malformed.bindings.descriptors.end(), + "native validation fixture lacks a buffer group"); + malformed_buffers->resources.clear(); + std::vector rejected_spirv = {0xdeadbeefu}; + const auto rejected_before = rejected_spirv; + std::string rejected_error; + Check(!ShaderRecompiler::Spirv::EmitProgram( + malformed, result.resources, nullptr, nullptr, nullptr, + rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("topology") != std::string::npos, + "malformed native binding plan was not rejected transactionally"); - auto malformed_inst = result.program; - auto* buffer_inst = FindBufferInstruction(&malformed_inst); - Check(buffer_inst != nullptr, "native validation fixture lacks a buffer instruction"); - buffer_inst->op = ShaderRecompiler::IR::Opcode::FlatLoadDword; - buffer_inst->memory.kind = ShaderRecompiler::IR::ResourceKind::Flat; - buffer_inst->src_count = 1; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(malformed_inst, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("contract") != std::string::npos, - "malformed native instruction was not rejected before fatal emitter paths"); + auto stale_resources = result.resources; + uint32_t float_storage = UINT32_MAX; + for (uint32_t i = 0; i < result.program.info.images.size(); i++) { + if (result.program.info.images[i].kind == + ShaderRecompiler::IR::ResourceKind::StorageImage) { + float_storage = i; + break; + } + } + Check(float_storage != UINT32_MAX, + "native validation fixture lacks float storage image"); + stale_resources.images[float_storage].dwords[1] = 20u << 20u; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram( + result.program, stale_resources, nullptr, nullptr, nullptr, + rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("specialized format") != std::string::npos, + "stale runtime image format re-selected an absent descriptor group"); - auto missing_buffer_operand = result.program; - buffer_inst = FindBufferInstruction(&missing_buffer_operand); - Check(buffer_inst != nullptr, "native validation fixture lacks a buffer-arity candidate"); - buffer_inst->src_count--; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(missing_buffer_operand, result.resources, nullptr, - nullptr, nullptr, rejected_spirv, - &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("contract") != std::string::npos, - "buffer instruction with a missing address operand was accepted"); + auto stale_dimension = result.resources; + stale_dimension.images[float_storage].dwords[3] &= 0x0fffffffu; + rejected_spirv = rejected_before; + rejected_error.clear(); + Check(!ShaderRecompiler::Spirv::EmitProgram( + result.program, stale_dimension, nullptr, nullptr, nullptr, + rejected_spirv, &rejected_error) && + rejected_spirv == rejected_before && + rejected_error.find("specialized dimension") != std::string::npos, + "unsupported runtime image type bypassed specialization validation"); - auto extra_buffer_operand = result.program; - buffer_inst = FindBufferInstruction(&extra_buffer_operand); - Check(buffer_inst != nullptr, "native validation fixture lacks an extra-operand candidate"); - buffer_inst->src_count++; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(extra_buffer_operand, result.resources, nullptr, - nullptr, nullptr, rejected_spirv, - &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("contract") != std::string::npos, - "buffer instruction with an extra address operand was accepted"); - - auto malformed_atomic = result.program; - auto* atomic_inst = FindBufferInstruction(&malformed_atomic); - Check(atomic_inst != nullptr, "native validation fixture lacks an atomic candidate"); - atomic_inst->op = ShaderRecompiler::IR::Opcode::AtomicAddU32; - atomic_inst->memory.kind = ShaderRecompiler::IR::ResourceKind::ScalarBuffer; - atomic_inst->src_count = 2; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check( - !ShaderRecompiler::Spirv::EmitProgram(malformed_atomic, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && rejected_error.find("atomic") != std::string::npos, - "malformed atomic instruction reached a fatal descriptor path"); - - auto malformed_float_atomic = result.program; - atomic_inst = FindBufferInstruction(&malformed_float_atomic); - Check(atomic_inst != nullptr, "native validation fixture lacks a float atomic candidate"); - atomic_inst->op = ShaderRecompiler::IR::Opcode::AtomicFMaxF32; - atomic_inst->memory.kind = ShaderRecompiler::IR::ResourceKind::Lds; - atomic_inst->memory.resource = 0; - atomic_inst->src_count = 2; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(malformed_float_atomic, result.resources, nullptr, - nullptr, nullptr, rejected_spirv, - &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("floating-point atomic") != std::string::npos, - "buffer-only floating-point atomic accepted a non-buffer resource"); - - auto invalid_ir_opcode = result.program; - Check(!invalid_ir_opcode.blocks.empty() && !invalid_ir_opcode.blocks.front().instructions.empty(), - "native validation fixture lacks an invalid-opcode candidate"); - invalid_ir_opcode.blocks.front().instructions.front().op = - ShaderRecompiler::IR::Opcode::Count; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(invalid_ir_opcode, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("opcode is out of range") != std::string::npos, - "out-of-range IR opcode was not rejected transactionally before emission"); - - auto overwide_atomic = result.program; - atomic_inst = FindBufferInstruction(&overwide_atomic); - Check(atomic_inst != nullptr, "native validation fixture lacks an overwide atomic candidate"); - atomic_inst->op = ShaderRecompiler::IR::Opcode::AtomicAddU32; - atomic_inst->src_count = 5; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(overwide_atomic, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("fixed IR storage") != std::string::npos, - "overwide atomic operands reached fixed emitter storage"); - - auto overwide_move = result.program; - auto& move_inst = overwide_move.blocks.front().instructions.front(); - move_inst.op = ShaderRecompiler::IR::Opcode::MoveU32; - move_inst.memory = {}; - move_inst.src_count = 5; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(overwide_move, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("fixed IR storage") != std::string::npos, - "overwide non-memory operands reached fixed emitter storage"); - - auto malformed_image = result.program; - auto* image_inst = static_cast(nullptr); - for (auto& block: malformed_image.blocks) { - const auto found = std::find_if( - block.instructions.begin(), block.instructions.end(), - [](const auto& inst) { return inst.op == ShaderRecompiler::IR::Opcode::ImageSample; }); - if (found != block.instructions.end()) { - image_inst = &*found; - break; - } - } - Check(image_inst != nullptr, "native validation fixture lacks a sampled image"); - image_inst->src_count = 0; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(malformed_image, result.resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("sampled image") != std::string::npos, - "malformed image operands reached fixed emitter storage"); - - auto stale_resources = result.resources; - uint32_t float_storage = UINT32_MAX; - for (uint32_t i = 0; i < result.program.info.images.size(); i++) { - if (result.program.info.images[i].kind == - ShaderRecompiler::IR::ResourceKind::StorageImage) { - float_storage = i; - break; - } - } - Check(float_storage != UINT32_MAX, "native validation fixture lacks float storage image"); - stale_resources.images[float_storage].dwords[1] = 20u << 20u; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(result.program, stale_resources, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("specialized format") != std::string::npos, - "stale runtime image format re-selected an absent descriptor group"); - - auto stale_dimension = result.resources; - stale_dimension.images[float_storage].dwords[3] &= 0x0fffffffu; - rejected_spirv = rejected_before; - rejected_error.clear(); - Check(!ShaderRecompiler::Spirv::EmitProgram(result.program, stale_dimension, nullptr, nullptr, - nullptr, rejected_spirv, &rejected_error) && - rejected_spirv == rejected_before && - rejected_error.find("specialized dimension") != std::string::npos, - "unsupported runtime image type bypassed specialization validation"); + ShaderRecompiler::IR::Inst *buffer_handle = nullptr; + for (auto *block : result.program.values->blocks) { + const auto found = std::ranges::find_if(*block, [](const auto &inst) { + return inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::GetBufferResource; + }); + if (found != block->end()) { + buffer_handle = &*found; + break; + } + } + Check(buffer_handle != nullptr, + "typed validation fixture lacks a buffer handle"); + const auto dense = buffer_handle->Flags(); + buffer_handle->SetFlags(UINT32_MAX); + rejected_spirv = rejected_before; + rejected_error.clear(); + const bool rejected = !ShaderRecompiler::Spirv::EmitProgram( + result.program, result.resources, nullptr, nullptr, nullptr, + rejected_spirv, &rejected_error); + buffer_handle->SetFlags(dense); + Check(rejected && rejected_spirv == rejected_before && + rejected_error.find("invalid dense resource") != std::string::npos, + "invalid typed buffer handle was not rejected transactionally"); } -bool LowerProvenanceOnly(const uint32_t* code, uint32_t words, ShaderRecompiler::IR::Program& ir, - std::string* error) { - ShaderRecompiler::Decoder::Program decoded; - if (!ShaderRecompiler::Decoder::DecodeProgram(std::span {code, words}, decoded, error)) { - return false; - } - ShaderRecompiler::CFG::Graph cfg; - if (!ShaderRecompiler::CFG::BuildGraph(decoded, cfg, error) || - !ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, 64, ir, error)) { - return false; - } - return ShaderRecompiler::IR::BuildScalarProvenance(ir, error) && - ShaderRecompiler::IR::BuildSrtPlan(ir, error); +bool LowerTypedPlanning(const uint32_t *code, uint32_t words, + ShaderRecompiler::IR::Program &ir, std::string *error) { + ShaderRecompiler::Decoder::Program decoded; + if (!ShaderRecompiler::Decoder::DecodeProgram(std::span{code, words}, decoded, + error)) { + return false; + } + ShaderRecompiler::CFG::Graph cfg; + if (!ShaderRecompiler::CFG::BuildGraph(decoded, cfg, error) || + !ShaderRecompiler::IR::LowerProgram(decoded, cfg, ShaderType::Compute, 64, + ir, error)) { + return false; + } + ShaderVertexInputInfo vertex; + ShaderPixelInputInfo pixel; + ShaderComputeInputInfo compute; + ir.values = std::make_shared(); + if (!ShaderRecompiler::Frontend::TranslateProgram(ir, *ir.values, &vertex, + &pixel, &compute, error)) { + return false; + } + ir.blocks.clear(); + ShaderRecompiler::IR::RewriteToSsa(ir.values->blocks); + ShaderRecompiler::IR::ConstantPropagationPass(ir.values->blocks); + for (auto &info : ir.values->block_info) { + info.condition = info.condition.Resolve(); + info.indirect_target = info.indirect_target.Resolve(); + } + ShaderRecompiler::IR::RemoveIdentities(ir.values->blocks); + ShaderRecompiler::IR::EliminateDeadCode(ir.values->blocks); + const auto read_lane = + ShaderRecompiler::IR::EliminateReadLane(*ir.values, ir.wave_size); + if (read_lane.rewritten_reads != 0) { + ShaderRecompiler::IR::ConstantPropagationPass(ir.values->blocks); + ShaderRecompiler::IR::RemoveIdentities(ir.values->blocks); + ShaderRecompiler::IR::EliminateDeadCode(ir.values->blocks); + } + if (!ShaderRecompiler::IR::BuildSrtPlan(ir, error)) { + return false; + } + ShaderRecompiler::IR::EliminateDeadCode(ir.values->blocks); + return true; } -bool ReadSrtHostDword(void*, uint64_t address, uint32_t* value) { - if (address == 0 || value == nullptr) { - return false; - } - std::memcpy(value, reinterpret_cast(address), sizeof(*value)); - return true; +const ShaderRecompiler::IR::DescriptorSource * +TypedDescriptorSource(const ShaderRecompiler::IR::Program &program, + uint32_t source) { + return program.values != nullptr && + source < program.values->descriptor_sources.size() + ? &program.values->descriptor_sources[source] + : nullptr; } -void TestScalarProvenanceRealWideMoveLowering() { - const uint32_t shader[] = { - EncodeSop1(0x04, 0, 20), // s_mov_b64 s[0:1], s[20:21] - EncodeSop1(0x04, 2, 22), // s_mov_b64 s[2:3], s[22:23] - EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), // buffer_load_dword via copied s[0:3] - EncodeSopp(0x01), - }; - - std::string error; - ShaderRecompiler::IR::Program ir; - const auto lowered = - LowerProvenanceOnly(shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); - - const ShaderRecompiler::IR::Instruction* use = nullptr; - for (const auto& block: ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::BufferLoadDword) { - use = &inst; - } - } - } - Check(use != nullptr, "real wide-move shader did not lower a buffer use"); - const auto* source = ShaderRecompiler::IR::GetDescriptorSource(ir, use->memory.resource_source); - Check(source != nullptr && - ShaderRecompiler::IR::DescriptorSourceResolved(ir, use->memory.resource_source), - "real wide-move descriptor source was unresolved"); - for (uint32_t i = 0; i < 4; i++) { - const auto& value = ir.provenance.values[source->dwords[i]]; - Check(value.op == ShaderRecompiler::IR::ScalarValueOp::UserData && value.imm == 20 + i, - "real s_mov_b64 lowering did not copy both descriptor pairs"); - } +bool ValueDependsOn(ShaderRecompiler::IR::Value value, + ShaderRecompiler::IR::ValueOpcode opcode, + std::vector &visited) { + value = value.Resolve(); + const auto *inst = value.TryInstruction(); + if (inst == nullptr) { + return false; + } + if (inst->GetOpcode() == opcode) { + return true; + } + if (std::ranges::find(visited, inst) != visited.end()) { + return false; + } + visited.push_back(inst); + for (size_t index = 0; index < inst->NumArgs(); index++) { + if (ValueDependsOn(inst->Arg(index), opcode, visited)) { + return true; + } + } + return false; } -void TestScalarProvenanceRealCarryAndScalarLoads() { - const uint32_t carry_shader[] = { - EncodeSop1(0x1f, 0, 0), // s_getpc_b64 s[0:1] - EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 - EncodeSop2(0x04, 1, 1, 128), // s_addc_u32 s1, s1, 0 - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 0, 1), // buffer_load_dword via computed s[0:3] - EncodeSopp(0x01), - }; - std::string error; - ShaderRecompiler::IR::Program carry_ir; - const auto carry_lowered = LowerProvenanceOnly( - carry_shader, static_cast(std::size(carry_shader)), carry_ir, &error); - Check(carry_lowered, error.c_str()); - const ShaderRecompiler::IR::DescriptorValue* carry_source = nullptr; - for (const auto& block: carry_ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::BufferLoadDword) { - carry_source = ShaderRecompiler::IR::GetDescriptorSource( - carry_ir, inst.memory.resource_source); - } - } - } - Check(carry_source != nullptr, "real add/addc descriptor source was not attached"); - const auto& high = carry_ir.provenance.values[carry_source->dwords[1]]; - Check(high.op == ShaderRecompiler::IR::ScalarValueOp::AddCarry && - carry_ir.provenance.values[high.args[2]].op == - ShaderRecompiler::IR::ScalarValueOp::Carry, - "real s_add_u32/s_addc_u32 lowering lost SCC carry provenance"); +bool ValueDependsOn(ShaderRecompiler::IR::Value value, + ShaderRecompiler::IR::ValueOpcode opcode) { + std::vector visited; + return ValueDependsOn(value, opcode, visited); +} - const uint32_t load_shader[] = { - EncodeSmem0(0x02, 0, 4), - 0u, // s_load_dwordx4 s[0:3] - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 0, 1), - EncodeSmem0(0x0a, 4, 4), - 0u, // s_buffer_load_dwordx4 s[4:7] - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 1, 1), - EncodeSopp(0x01), - }; - ShaderRecompiler::IR::Program load_ir; - const auto load_lowered = LowerProvenanceOnly( - load_shader, static_cast(std::size(load_shader)), load_ir, &error); - Check(load_lowered, error.c_str()); - uint32_t uses = 0; - for (const auto& block: load_ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op != ShaderRecompiler::IR::Opcode::BufferLoadDword) { - continue; - } - const auto* source = - ShaderRecompiler::IR::GetDescriptorSource(load_ir, inst.memory.resource_source); - Check(source != nullptr, "real scalar-load descriptor source was not attached"); - const auto expected = uses++ == 0 - ? ShaderRecompiler::IR::ScalarValueOp::ReadConst - : ShaderRecompiler::IR::ScalarValueOp::ReadConstBuffer; - for (uint32_t i = 0; i < 4; i++) { - Check(load_ir.provenance.values[source->dwords[i]].op == expected, - "real scalar load used the wrong provenance operation"); - } - } - } - Check(uses == 2, "real scalar-load provenance test did not find both buffer uses"); +void CheckFlattenedReadSlots(const ShaderRecompiler::IR::Program &program, + uint32_t expected, const char *message) { + std::vector slots; + for (const auto *block : program.values->blocks) { + for (const auto &inst : *block) { + if (inst.GetOpcode() == ShaderRecompiler::IR::ValueOpcode::ReadConst) { + const auto slot = inst.Arg(1).Resolve(); + Check(slot.IsImmediate() && + slot.GetType() == ShaderRecompiler::IR::Type::U32, + message); + slots.push_back(slot.U32()); + } + } + } + std::ranges::sort(slots); + Check(slots.size() == expected, message); + for (uint32_t index = 0; index < expected; index++) { + Check(slots[index] == index, message); + } +} - const uint32_t inline_sampler_shader[] = { - EncodeSop2(0x25, 12, 128 + 12, 128 + 44), // s_bfm_b64 s[12:13], 12, 44 - EncodeSop1(0x04, 14, 255), - 0x09500000u, // s_mov_b64 s[14:15], 0x09500000 - EncodeMimg0(0xa0, 0xf), - EncodeMimg1(2, 1, 3, 6), // image_sample_a v2, v6, s[4:11], s[12:15] - EncodeSopp(0x01), - }; - ShaderRecompiler::IR::Program inline_sampler_ir; - error.clear(); - Check(LowerProvenanceOnly(inline_sampler_shader, - static_cast(std::size(inline_sampler_shader)), - inline_sampler_ir, &error), - error.c_str()); - uint32_t sampler_source = 0; - for (const auto& block: inline_sampler_ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::ImageSample) { - sampler_source = inst.memory.sampler_source; - } - } - } - ShaderRecompiler::IR::DescriptorValue sampler; - ShaderRecompiler::IR::SrtRuntime runtime; - Check(sampler_source >= 2 && - ShaderRecompiler::IR::DescriptorSourceResolved(inline_sampler_ir, sampler_source) && - ShaderRecompiler::IR::EvaluateDescriptorSource(inline_sampler_ir, sampler_source, - 0x10, runtime, sampler, &error) && - sampler.dwords[0] == 0 && sampler.dwords[1] == 0x00fff000u && - sampler.dwords[2] == 0x09500000u && sampler.dwords[3] == 0, - "real inline sampler construction was unresolved or evaluated incorrectly"); +bool ReadSrtHostDword(void *, uint64_t address, uint32_t *value) { + if (address == 0 || value == nullptr) { + return false; + } + std::memcpy(value, reinterpret_cast(address), sizeof(*value)); + return true; +} - auto user_data = ImageTestUserData(); - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Pixel; - options.user_data = user_data.data(); - ShaderRecompiler::CompileResult result; - error.clear(); - Check(ShaderRecompiler::TryRecompile(inline_sampler_shader, options, result, &error), - error.c_str()); +void TestTypedDescriptorRealWideMoveLowering() { + const uint32_t shader[] = { + EncodeSop1(0x04, 0, 20), // s_mov_b64 s[0:1], s[20:21] + EncodeSop1(0x04, 2, 22), // s_mov_b64 s[2:3], s[22:23] + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 0, 1), // buffer_store_dword via copied s[0:3] + EncodeSopp(0x01), + }; + + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = LowerTypedPlanning( + shader, static_cast(std::size(shader)), ir, &error); + Check(lowered, error.c_str()); + Check(ShaderRecompiler::IR::TrackResources(ir, &error), error.c_str()); + Check(ir.info.buffers.size() == 1, + "real wide-move shader did not track one buffer use"); + const auto *source = TypedDescriptorSource(ir, ir.info.buffers[0].source); + Check(source != nullptr && source->dword_count == 4, + "real wide-move descriptor source was unresolved"); + for (uint32_t i = 0; i < 4; i++) { + const auto *value = source->dwords[i].ResolveInstruction(); + Check(value != nullptr && + value->GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::GetUserData && + ShaderRecompiler::IR::RegIndex(value->Arg(0).ScalarRegister()) == + 20 + i, + "real s_mov_b64 lowering did not copy both descriptor pairs"); + } +} + +void TestTypedDescriptorRealCarryAndScalarLoads() { + const uint32_t carry_shader[] = { + EncodeSop1(0x1f, 0, 0), // s_getpc_b64 s[0:1] + EncodeSop2(0x00, 0, 0, 129), // s_add_u32 s0, s0, 1 + EncodeSop2(0x04, 1, 1, 128), // s_addc_u32 s1, s1, 0 + EncodeSMovB32(2, 128), // descriptor dword 2 = 0 + EncodeSMovB32(3, 128), // descriptor dword 3 = 0 + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 0, 1), // buffer_store_dword via computed s[0:3] + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program carry_ir; + const auto carry_lowered = LowerTypedPlanning( + carry_shader, static_cast(std::size(carry_shader)), carry_ir, + &error); + Check(carry_lowered, error.c_str()); + const ShaderRecompiler::IR::Inst *carry_handle = nullptr; + for (const auto *block : carry_ir.values->blocks) { + for (const auto &inst : *block) { + if (inst.GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::StoreBufferU32) { + carry_handle = inst.Arg(0).ResolveInstruction(); + } + } + } + Check(carry_handle != nullptr && + carry_handle->GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::GetBufferResource && + carry_handle->NumArgs() == 4, + "real add/addc descriptor source was not attached"); + ShaderRecompiler::IR::DescriptorSource carry_descriptor; + carry_descriptor.dword_count = 4; + for (uint32_t index = 0; index < 4; index++) { + carry_descriptor.dwords[index] = carry_handle->Arg(index).Resolve(); + } + const auto carry_source_index = + static_cast(carry_ir.values->descriptor_sources.size()); + carry_ir.values->descriptor_sources.push_back(carry_descriptor); + const auto *carry_source = + TypedDescriptorSource(carry_ir, carry_source_index); + Check(carry_source != nullptr && carry_source->dword_count == 4 && + ValueDependsOn(carry_source->dwords[0], + ShaderRecompiler::IR::ValueOpcode::GetShaderBase) && + ValueDependsOn(carry_source->dwords[1], + ShaderRecompiler::IR::ValueOpcode::GetShaderBase) && + ValueDependsOn(carry_source->dwords[1], + ShaderRecompiler::IR::ValueOpcode::IAddCarry32), + "real s_add_u32/s_addc_u32 lowering lost SCC carry provenance"); + std::array carry_user_data{}; + const uint64_t shader_base = 0x00000012fffffffbull; + const uint64_t expected_pc = shader_base + 5u; + ShaderRecompiler::IR::SrtRuntime carry_runtime{carry_user_data, shader_base, + nullptr, nullptr}; + ShaderRecompiler::IR::DescriptorValue carry_value; + Check(ShaderRecompiler::IR::EvaluateDescriptorSource( + carry_ir, carry_source_index, 0, carry_runtime, carry_value, + &error) && + carry_value.dwords[0] == static_cast(expected_pc) && + carry_value.dwords[1] == static_cast(expected_pc >> 32u), + "S_GETPC shader-base or add/addc carry evaluation was incorrect"); + + const uint32_t load_shader[] = { + EncodeSmem0(0x02, 0, 4), + 125u << 25u, // s_load_dwordx4 s[0:3], null SOFFSET + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 0, 1), + EncodeSmem0(0x0a, 4, 4), + 125u << 25u, // s_buffer_load_dwordx4 s[4:7], null SOFFSET + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 1, 1), + EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program load_ir; + const auto load_lowered = LowerTypedPlanning( + load_shader, static_cast(std::size(load_shader)), load_ir, + &error); + Check(load_lowered, error.c_str()); + Check(load_ir.values->srt_reads.size() == 8 && + load_ir.values->dynamic_reads.empty(), + "real scalar loads did not build eight flattened reads"); + uint32_t address_reads = 0; + uint32_t buffer_reads = 0; + for (const auto &read : load_ir.values->srt_reads) { + const auto *raw = read.value.ResolveInstruction(); + Check(raw != nullptr, + "real scalar-load SRT entry lost its raw typed value"); + address_reads += + raw->GetOpcode() == ShaderRecompiler::IR::ValueOpcode::LoadAddressU32; + buffer_reads += + raw->GetOpcode() == ShaderRecompiler::IR::ValueOpcode::ReadConstBuffer; + } + Check(address_reads == 4 && buffer_reads == 4, + "real scalar loads used the wrong raw typed operations"); + Check(ShaderRecompiler::IR::TrackResources(load_ir, &error), error.c_str()); + Check(load_ir.info.buffers.size() == 2, + "real scalar-load descriptor sources were not attached"); + for (const auto &buffer : load_ir.info.buffers) { + const auto *source = TypedDescriptorSource(load_ir, buffer.source); + Check(source != nullptr && source->dword_count == 4, + "real scalar-load descriptor source was not attached"); + for (uint32_t i = 0; i < 4; i++) { + const auto *value = source->dwords[i].ResolveInstruction(); + Check(value != nullptr && + value->GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::ReadConst, + "real scalar load was not flattened in the descriptor source"); + } + } + + const uint32_t inline_sampler_shader[] = { + EncodeSop2(0x25, 12, 128 + 12, 128 + 44), // s_bfm_b64 s[12:13], 12, 44 + EncodeSop1(0x04, 14, 255), + 0x09500000u, // s_mov_b64 s[14:15], 0x09500000 + EncodeMimg0(0xa0, 0xf), + EncodeMimg1(2, 1, 3, 6), // image_sample_a v2, v6, s[4:11], s[12:15] + EncodeMubuf0(0x1c), + EncodeMubuf1(2, 1, + 1), // keep the sampled value live through a buffer store + EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program inline_sampler_ir; + error.clear(); + Check(LowerTypedPlanning( + inline_sampler_shader, + static_cast(std::size(inline_sampler_shader)), + inline_sampler_ir, &error) && + ShaderRecompiler::IR::TrackResources(inline_sampler_ir, &error), + error.c_str()); + ShaderRecompiler::IR::DescriptorValue sampler; + ShaderRecompiler::IR::SrtRuntime runtime; + Check(inline_sampler_ir.info.samplers.size() == 1 && + TypedDescriptorSource(inline_sampler_ir, + inline_sampler_ir.info.samplers[0].source) != + nullptr && + ShaderRecompiler::IR::EvaluateDescriptorSource( + inline_sampler_ir, inline_sampler_ir.info.samplers[0].source, + 0x10, runtime, sampler, &error) && + sampler.dwords[0] == 0 && sampler.dwords[1] == 0x00fff000u && + sampler.dwords[2] == 0x09500000u && sampler.dwords[3] == 0, + "real inline sampler construction was unresolved or evaluated " + "incorrectly"); + + auto user_data = ImageTestUserData(); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Pixel; + options.user_data = user_data.data(); + ShaderRecompiler::CompileResult result; + error.clear(); + Check(ShaderRecompiler::TryRecompile(inline_sampler_shader, options, result, + &error), + error.c_str()); } void TestSrtWalkerRealSmemLowering() { - const uint32_t shader[] = { - EncodeSMovB32(124, 130), // m0 = 2 - EncodeSmem0(0x02, 0, 4), - (124u << 25u) | 2u, // s_load_dwordx4 s[0:3], s[8:9], m0 offset + immediate 2 - EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), - }; - std::string error; - ShaderRecompiler::IR::Program ir; - const auto lowered = - LowerProvenanceOnly(shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); - Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), - "real SMEM lowering did not build four compact SRT reads"); + const uint32_t shader[] = { + EncodeSMovB32(124, 130), // m0 = 2 + EncodeSmem0(0x02, 0, 4), + (124u << 25u) | 2u, // s_load_dwordx4 s[0:3], s[8:9], m0 offset + + // immediate 2 + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 0, 1), + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = LowerTypedPlanning( + shader, static_cast(std::size(shader)), ir, &error); + Check(lowered, error.c_str()); + Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), + "real SMEM lowering did not build four compact SRT reads"); - const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; - std::array user_data = {}; - const auto address = reinterpret_cast(table.data()); - user_data[8] = static_cast(address); - user_data[9] = static_cast(address >> 32u); - std::vector flat; - const ShaderRecompiler::IR::SrtRuntime runtime {user_data, 0, ReadSrtHostDword, nullptr}; - const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error); - Check(walked, error.c_str()); - Check(flat.size() == table.size() && std::equal(flat.begin(), flat.end(), table.begin()), - "real SMEM SRT walk did not apply component-level alignment"); - Check(ShaderRecompiler::IR::PatchSrtReads(ir, &error), error.c_str()); - uint32_t patched = 0; - for (const auto& block: ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { - Check(inst.src[0].imm == patched++, "real SMEM patch used the wrong flat offset"); - } - } - } - Check(patched == 4, "real SMEM patch did not rewrite every component"); + const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u}; + std::array user_data = {}; + const auto address = reinterpret_cast(table.data()); + user_data[8] = static_cast(address); + user_data[9] = static_cast(address >> 32u); + std::vector flat; + const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, + nullptr}; + const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error); + Check(walked, error.c_str()); + Check(flat.size() == table.size() && + std::equal(flat.begin(), flat.end(), table.begin()), + "real SMEM SRT walk did not apply component-level alignment"); + CheckFlattenedReadSlots(ir, 4, "real SMEM patch used the wrong flat offsets"); +} + +void TestSrtWalkerVccBaseLowering() { + const uint32_t shader[] = { + EncodeSMovB32(106, 27), EncodeSMovB32(107, 28), + EncodeSmem0(0x02, 0, 53), 125u << 25u, + EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1), + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), ir, + &error), + error.c_str()); + Check(ir.values->srt_reads.size() == 4, + "VCC-based SMEM lowering did not build four SRT reads"); + + const std::array table = {0x11111111u, 0x22222222u, + 0x33333333u, 0x44444444u}; + std::array user_data{}; + const auto address = reinterpret_cast(table.data()); + user_data[27] = static_cast(address); + user_data[28] = static_cast(address >> 32u); + const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, + nullptr}; + std::vector flat; + Check(ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error), + error.c_str()); + Check(flat.size() == table.size() && + std::equal(flat.begin(), flat.end(), table.begin()), + "typed SSA lost an SMEM base copied through VCC"); } void TestSrtWalkerRealSBufferLowering() { - const uint32_t shader[] = { - EncodeSMovB32(124, 130), // m0 = 2 - EncodeSmem0(0x0a, 0, 4), - (124u << 25u) | 2u, // s_buffer_load_dwordx4 s[0:3], s[8:11], m0 + 2 - EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), - }; - std::string error; - ShaderRecompiler::IR::Program ir; - const auto lowered = - LowerProvenanceOnly(shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); - Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), - "real S_BUFFER_LOAD lowering did not build four compact reads"); + const uint32_t shader[] = { + EncodeSMovB32(124, 130), // m0 = 2 + EncodeSmem0(0x0a, 0, 4), + (124u << 25u) | 2u, // s_buffer_load_dwordx4 s[0:3], s[8:11], m0 + 2 + EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = LowerTypedPlanning( + shader, static_cast(std::size(shader)), ir, &error); + Check(lowered, error.c_str()); + Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), + "real S_BUFFER_LOAD lowering did not build four compact reads"); - const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u, - 0x55555555u}; - std::array user_data = {}; - const auto address = reinterpret_cast(table.data()); - user_data[8] = static_cast(address); - user_data[9] = static_cast(address >> 32u); - user_data[10] = sizeof(table); - std::vector flat; - const ShaderRecompiler::IR::SrtRuntime runtime {user_data, 0, ReadSrtHostDword, nullptr}; - const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error); - Check(walked, error.c_str()); - Check(flat.size() == 4 && std::equal(flat.begin(), flat.end(), table.begin() + 1), - "real S_BUFFER_LOAD walk used the wrong final alignment"); + const std::array table = {0x11111111u, 0x22222222u, 0x33333333u, + 0x44444444u, 0x55555555u}; + std::array user_data = {}; + const auto address = reinterpret_cast(table.data()); + user_data[8] = static_cast(address); + user_data[9] = static_cast(address >> 32u); + user_data[10] = sizeof(table); + std::vector flat; + const ShaderRecompiler::IR::SrtRuntime runtime{user_data, 0, ReadSrtHostDword, + nullptr}; + const auto walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error); + Check(walked, error.c_str()); + Check(flat.size() == 4 && + std::equal(flat.begin(), flat.end(), table.begin() + 1), + "real S_BUFFER_LOAD walk used the wrong final alignment"); - user_data[10] = 4 * sizeof(uint32_t); - const auto flat_before_failure = flat; - const auto bounds_walked = ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error); - Check(!bounds_walked && error.find("exceeds size=16") != std::string::npos, - "real S_BUFFER_LOAD walk ignored descriptor bounds"); - Check(flat == flat_before_failure, - "failed real S_BUFFER_LOAD walk changed the prior flat snapshot"); - Check(ShaderRecompiler::IR::PatchSrtReads(ir, &error), error.c_str()); - uint32_t patched = 0; - for (const auto& block: ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { - Check(inst.src[0].imm == patched++, - "real S_BUFFER_LOAD patch used the wrong flat offset"); - } - } - } - Check(patched == 4, "real S_BUFFER_LOAD patch did not rewrite every component"); + user_data[10] = 4 * sizeof(uint32_t); + const auto flat_before_failure = flat; + const auto bounds_walked = + ShaderRecompiler::IR::WalkSrt(ir, runtime, flat, &error); + Check(!bounds_walked && error.find("exceeds size 16") != std::string::npos, + "real S_BUFFER_LOAD walk ignored descriptor bounds"); + Check(flat == flat_before_failure, + "failed real S_BUFFER_LOAD walk changed the prior flat snapshot"); + CheckFlattenedReadSlots( + ir, 4, "real S_BUFFER_LOAD patch used the wrong flat offsets"); - const uint32_t negative_shader[] = { - EncodeSmem0(0x0a, 0, 4), - (125u << 25u) | 0x1ffffeu, // illegal negative S_BUFFER immediate - EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), - }; - ShaderRecompiler::IR::Program negative_ir; - const auto negative_lowered = LowerProvenanceOnly( - negative_shader, static_cast(std::size(negative_shader)), negative_ir, &error); - Check(negative_lowered, error.c_str()); - user_data[10] = sizeof(table); - Check(!ShaderRecompiler::IR::WalkSrt(negative_ir, runtime, flat, &error) && - error.find("negative immediate") != std::string::npos, - "real S_BUFFER_LOAD walk accepted a negative immediate"); + const uint32_t negative_shader[] = { + EncodeSmem0(0x0a, 0, 4), + (125u << 25u) | 0x1ffffeu, // illegal negative S_BUFFER immediate + EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01), + }; + ShaderRecompiler::IR::Program negative_ir; + const auto negative_lowered = LowerTypedPlanning( + negative_shader, static_cast(std::size(negative_shader)), + negative_ir, &error); + Check(negative_lowered, error.c_str()); + user_data[10] = sizeof(table); + Check(!ShaderRecompiler::IR::WalkSrt(negative_ir, runtime, flat, &error) && + error.find("negative immediate") != std::string::npos, + "real S_BUFFER_LOAD walk accepted a negative immediate"); } void TestScalarMemoryLoadsSnapshotOverlappingOperands() { - const auto CheckOverlap = [](uint32_t opcode) { - const uint32_t shader[] = { - EncodeSmem0(opcode, 0, 0), - 125u << 25u, // load s[0:3] through overlapping s[0:*], null SOFFSET - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 0, 1), - EncodeSopp(0x01), - }; - std::string error; - ShaderRecompiler::IR::Program ir; - Check(LowerProvenanceOnly(shader, static_cast(std::size(shader)), ir, &error), - error.c_str()); - Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), - opcode == 0x02 - ? "overlapping S_LOAD operands were evaluated after a component write" - : "overlapping S_BUFFER_LOAD operands were evaluated after a component write"); - Check(ShaderRecompiler::IR::PatchSrtReads(ir, &error), error.c_str()); - uint32_t patched = 0; - for (const auto& block: ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { - Check(inst.src[0].imm == patched++, - "overlapping scalar-memory patch used the wrong flat offset"); - } - } - } - Check(patched == 4, "overlapping scalar-memory load was not fully patched"); - }; - CheckOverlap(0x02); // s_load_dwordx4 - CheckOverlap(0x0a); // s_buffer_load_dwordx4 + const auto CheckOverlap = [](uint32_t opcode) { + const uint32_t shader[] = { + EncodeSmem0(opcode, 0, 0), + 125u << 25u, // load s[0:3] through overlapping s[0:*], null SOFFSET + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 0, 1), + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), + ir, &error), + error.c_str()); + Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), + opcode == 0x02 ? "overlapping S_LOAD operands were evaluated after a " + "component write" + : "overlapping S_BUFFER_LOAD operands were evaluated " + "after a component write"); + CheckFlattenedReadSlots( + ir, 4, "overlapping scalar-memory patch used the wrong flat offsets"); + }; + CheckOverlap(0x02); // s_load_dwordx4 + CheckOverlap(0x0a); // s_buffer_load_dwordx4 } void TestScalarMemoryLoadCrossesIntoVcc() { - const uint32_t shader[] = { - EncodeSmem0(0x02, 104, 4), - 125u << 25u, // s_load_dwordx4 s[104:105], vcc_lo, vcc_hi - EncodeMubuf0(0x0c), - EncodeMubuf1(0, 26, 1), - EncodeSopp(0x01), - }; - std::string error; - ShaderRecompiler::IR::Program ir; - Check(LowerProvenanceOnly(shader, static_cast(std::size(shader)), ir, &error), - error.c_str()); - Check(ir.srt.reads.size() == 4 && ir.srt.dynamic_reads.empty(), - "wide SMEM destination crossing into VCC lost scalar provenance"); - Check(ShaderRecompiler::IR::PatchSrtReads(ir, &error), error.c_str()); - uint32_t patched = 0; - for (const auto& block: ir.blocks) { - for (const auto& inst: block.instructions) { - if (inst.op == ShaderRecompiler::IR::Opcode::LoadSrtDword) { - Check(inst.src[0].imm == patched++, - "wide SMEM destination crossing into VCC used the wrong flat offset"); - } - } - } - Check(patched == 4, "wide SMEM destination crossing into VCC was not fully patched"); + const uint32_t shader[] = { + EncodeSmem0(0x02, 104, 4), + 125u << 25u, // s_load_dwordx4 s[104:105], vcc_lo, vcc_hi + EncodeSopp(0x07, + 2), // consume both VCC dwords and skip the store when nonzero + EncodeMubuf0(0x1c), + EncodeMubuf1(0, 26, 1), + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerTypedPlanning(shader, static_cast(std::size(shader)), ir, + &error), + error.c_str()); + Check(ir.values->srt_reads.size() == 4 && ir.values->dynamic_reads.empty(), + "wide SMEM destination crossing into VCC lost scalar provenance"); + CheckFlattenedReadSlots( + ir, 4, "wide SMEM destination crossing into VCC used wrong flat offsets"); + Check(ShaderRecompiler::IR::TrackResources(ir, &error), error.c_str()); + Check(ir.info.buffers.size() == 1, + "wide SMEM destination crossing into VCC lost its buffer use"); + const auto *source = + TypedDescriptorSource(ir, ir.info.buffers.front().source); + Check(source != nullptr && source->dword_count == 4, + "wide SMEM destination crossing into VCC lost its descriptor source"); + for (uint32_t dword = 0; dword < 4; dword++) { + const auto *value = source->dwords[dword].ResolveInstruction(); + Check(value != nullptr && + value->GetOpcode() == + ShaderRecompiler::IR::ValueOpcode::ReadConst, + "wide SMEM descriptor lost a dword crossing into VCC"); + } } void TestResourceTrackingRealDensePatching() { - const uint32_t shader[] = { - EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), // buffer load s[0:3] - EncodeMubuf0(0x0c), EncodeMubuf1(1, 0, 1), // same buffer - EncodeMubuf0(0x1c), EncodeMubuf1(2, 1, 1), // buffer store s[4:7] - EncodeMimg0(0x20, 0xf), EncodeMimg1(8, 0, 2, 1), // sampled image s[0:7], sampler s[8:11] - EncodeMimg0(0x08, 0xf), EncodeMimg1(12, 0, 0, 1), // storage view of the same image source - EncodeSopp(0x01), - }; - std::string error; - ShaderRecompiler::IR::Program ir; - const auto lowered = - LowerProvenanceOnly(shader, static_cast(std::size(shader)), ir, &error); - Check(lowered, error.c_str()); - Check(ShaderRecompiler::IR::PatchSrtReads(ir, &error), error.c_str()); - const auto tracked = ShaderRecompiler::IR::TrackResources(ir, &error); - Check(tracked, error.c_str()); - Check(ir.info.buffers.size() == 2 && ir.info.images.size() == 2 && ir.info.samplers.size() == 1, - "real resource tracking produced the wrong dense list sizes"); + const uint32_t shader[] = { + EncodeMubuf0(0x0c), + EncodeMubuf1(0, 0, 1), // buffer load s[0:3] + EncodeMubuf0(0x0c), + EncodeMubuf1(1, 0, 1), // same buffer + EncodeMubuf0(0x1d), + EncodeMubuf1(0, 1, 1), // keep both loads live in s[4:7] + EncodeMimg0(0x20, 0xf), + EncodeMimg1(8, 0, 2, 1), // sampled image s[0:7], sampler s[8:11] + EncodeMimg0(0x08, 0xf), + EncodeMimg1(8, 0, 0, 1), // keep sample live in storage view + EncodeSopp(0x01), + }; + std::string error; + ShaderRecompiler::IR::Program ir; + const auto lowered = LowerTypedPlanning( + shader, static_cast(std::size(shader)), ir, &error); + Check(lowered, error.c_str()); + const auto tracked = ShaderRecompiler::IR::TrackResources(ir, &error); + Check(tracked, error.c_str()); + Check(ir.info.buffers.size() == 2 && ir.info.images.size() == 2 && + ir.info.samplers.size() == 1, + "real resource tracking produced the wrong dense list sizes"); - uint32_t buffer_use = 0; - uint32_t image_use = 0; - for (const auto& block: ir.blocks) { - for (const auto& inst: block.instructions) { - switch (inst.memory.kind) { - case ShaderRecompiler::IR::ResourceKind::Buffer: { - const uint32_t expected = buffer_use++ < 2 ? 0 : 1; - Check(inst.memory.resource == expected, - "real buffer operand was not patched densely"); - break; - } - case ShaderRecompiler::IR::ResourceKind::Image: - case ShaderRecompiler::IR::ResourceKind::StorageImage: - Check(inst.memory.resource == image_use++, - "real image operand was not patched by view class"); - if (inst.op == ShaderRecompiler::IR::Opcode::ImageSample) { - Check(inst.memory.sampler == 0, - "real sampler operand was not patched densely"); - } - break; - default: break; - } - } - } - Check(buffer_use == 3 && image_use == 2 && ir.info.buffers[0].read && - ir.info.buffers[1].written, - "real tracked resource access facts were incomplete"); + uint32_t buffer_use = 0; + uint32_t image_use = 0; + for (const auto *block : ir.values->blocks) { + for (const auto &inst : *block) { + const auto op = inst.GetOpcode(); + switch (op) { + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU8: + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU16: + case ShaderRecompiler::IR::ValueOpcode::LoadBufferU32: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU8: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU16: + case ShaderRecompiler::IR::ValueOpcode::StoreBufferU32: { + const auto &memory = + ir.values->memory_info + [inst.Flags().index]; + const uint32_t expected = buffer_use++ < 2 ? 0 : 1; + const auto *handle = inst.Arg(0).ResolveInstruction(); + Check(memory.resource == expected && handle != nullptr && + handle->Flags() == expected, + "real buffer operand was not patched densely"); + break; + } + case ShaderRecompiler::IR::ValueOpcode::ImageSampleRaw: + case ShaderRecompiler::IR::ValueOpcode::ImageWrite: { + const auto &memory = + ir.values->memory_info + [inst.Flags().index]; + const auto expected = image_use++; + const auto *handle = inst.Arg(0).ResolveInstruction(); + Check(memory.resource == expected && handle != nullptr && + handle->Flags() == expected, + "real image operand was not patched by view class"); + if (op == ShaderRecompiler::IR::ValueOpcode::ImageSampleRaw) { + const auto *sampler = inst.Arg(1).ResolveInstruction(); + Check(memory.sampler == 0 && sampler != nullptr && + sampler->Flags() == 0, + "real sampler operand was not patched densely"); + } + break; + } + default: + break; + } + } + } + Check(buffer_use == 4 && image_use == 2 && ir.info.buffers[0].read && + ir.info.buffers[1].written, + "real tracked resource access facts were incomplete"); } void TestLowerProgramResetsAnalysisState() { - const uint32_t first_shader[] = {EncodeMubuf0(0x0c), EncodeMubuf1(0, 0, 1), EncodeSopp(0x01)}; - std::string error; - ShaderRecompiler::IR::Program ir; - Check(LowerProvenanceOnly(first_shader, static_cast(std::size(first_shader)), ir, - &error), - error.c_str()); - Check(ShaderRecompiler::IR::PatchSrtReads(ir, &error), error.c_str()); - Check(ShaderRecompiler::IR::TrackResources(ir, &error), error.c_str()); - ShaderComputeInputInfo compute; - Check(ShaderRecompiler::IR::CollectShaderInfo(ir, {.compute = &compute}, &error), - error.c_str()); - Check(ir.resource_tracking_complete && ir.shader_info_complete && !ir.info.buffers.empty(), - "analysis-reset fixture did not reach completed state"); - ir.shader_hash = 0xdeadbeef; + const uint32_t first_shader[] = {EncodeMubuf0(0x1c), EncodeMubuf1(0, 0, 1), + EncodeSopp(0x01)}; + std::string error; + ShaderRecompiler::IR::Program ir; + Check(LowerTypedPlanning(first_shader, + static_cast(std::size(first_shader)), ir, + &error), + error.c_str()); + Check(ShaderRecompiler::IR::TrackResources(ir, &error), error.c_str()); + ShaderComputeInputInfo compute; + Check(ShaderRecompiler::IR::CollectShaderInfo(ir, {.compute = &compute}, + &error), + error.c_str()); + Check(ir.resource_tracking_complete && ir.shader_info_complete && + !ir.info.buffers.empty(), + "analysis-reset fixture did not reach completed state"); + ir.shader_hash = 0xdeadbeef; - const uint32_t second_shader[] = {EncodeSopp(0x01)}; - Check(LowerProvenanceOnly(second_shader, static_cast(std::size(second_shader)), ir, - &error), - error.c_str()); - Check(!ir.resource_tracking_complete && !ir.shader_info_complete && ir.srt_plan_complete && - ir.shader_hash == 0 && ir.info.buffers.empty() && ir.info.images.empty() && - ir.info.samplers.empty() && ir.info.sampled_pairs.empty() && ir.info.inputs.empty() && - ir.info.outputs.empty(), - "LowerProgram reused stale provenance/resource/interface state"); + const uint32_t second_shader[] = {EncodeSopp(0x01)}; + Check(LowerTypedPlanning(second_shader, + static_cast(std::size(second_shader)), ir, + &error), + error.c_str()); + Check(!ir.resource_tracking_complete && !ir.shader_info_complete && + ir.srt_plan_complete && ir.values != nullptr && + ir.values->srt_reads.empty() && ir.shader_hash == 0 && + ir.info.buffers.empty() && ir.info.images.empty() && + ir.info.samplers.empty() && ir.info.sampled_pairs.empty() && + ir.info.inputs.empty() && ir.info.outputs.empty(), + "LowerProgram reused stale provenance/resource/interface state"); } void TestNewShaderRecompilerStageInputInfo() { - using StageInputKind = ShaderRecompiler::IR::StageInputKind; + using StageInputKind = ShaderRecompiler::IR::StageInputKind; - const uint32_t shader[] = {0xbf810000u}; + const uint32_t shader[] = {0xbf810000u}; - ShaderComputeInputInfo cs_info {}; - cs_info.group_id[0] = true; - cs_info.thread_ids_num = 3; - cs_info.dispatch_thread_dimensions = true; + ShaderComputeInputInfo cs_info{}; + cs_info.group_id[0] = true; + cs_info.thread_ids_num = 3; + cs_info.dispatch_thread_dimensions = true; - ShaderRecompiler::CompileOptions cs_options; - cs_options.stage = ShaderType::Compute; - cs_options.compute_input_info = &cs_info; + ShaderRecompiler::CompileOptions cs_options; + cs_options.stage = ShaderType::Compute; + cs_options.compute_input_info = &cs_info; - ShaderRecompiler::CompileResult cs_result; - std::string error; - Check(ShaderRecompiler::TryRecompile(shader, cs_options, cs_result, &error), error.c_str()); - Check(ProgramHasInput(cs_result.program, StageInputKind::WorkgroupId), - "compute WorkgroupId input missing from reflection"); - Check(ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationId), - "compute LocalInvocationId input missing from reflection"); - Check(ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationIndex), - "compute LocalInvocationIndex input missing from reflection"); - Check(ProgramHasInput(cs_result.program, StageInputKind::GlobalInvocationId), - "compute GlobalInvocationId input missing from reflection"); - Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 26u), - "SPIR-V lacks WorkgroupId BuiltIn decoration"); - Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 27u), - "SPIR-V lacks LocalInvocationId BuiltIn decoration"); - Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 28u), - "SPIR-V lacks GlobalInvocationId BuiltIn decoration"); - Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 29u), - "SPIR-V lacks LocalInvocationIndex BuiltIn decoration"); - CheckSpirvBinaryValidates(cs_result.spirv); + ShaderRecompiler::CompileResult cs_result; + std::string error; + Check(ShaderRecompiler::TryRecompile(shader, cs_options, cs_result, &error), + error.c_str()); + Check(ProgramHasInput(cs_result.program, StageInputKind::WorkgroupId), + "compute WorkgroupId input missing from reflection"); + Check(ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationId), + "compute LocalInvocationId input missing from reflection"); + Check( + ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationIndex), + "compute LocalInvocationIndex input missing from reflection"); + Check(ProgramHasInput(cs_result.program, StageInputKind::GlobalInvocationId), + "compute GlobalInvocationId input missing from reflection"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 26u), + "SPIR-V lacks WorkgroupId BuiltIn decoration"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 27u), + "SPIR-V lacks LocalInvocationId BuiltIn decoration"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 28u), + "SPIR-V lacks GlobalInvocationId BuiltIn decoration"); + Check(SpirvHasDecorationValue(cs_result.spirv, 11u, 29u), + "SPIR-V lacks LocalInvocationIndex BuiltIn decoration"); + CheckSpirvBinaryValidates(cs_result.spirv); - ShaderRecompiler::CompileOptions vs_options; - vs_options.stage = ShaderType::Vertex; + ShaderRecompiler::CompileOptions vs_options; + vs_options.stage = ShaderType::Vertex; - ShaderRecompiler::CompileResult vs_result; - Check(ShaderRecompiler::TryRecompile(shader, vs_options, vs_result, &error), error.c_str()); - Check(ProgramHasInput(vs_result.program, StageInputKind::VertexIndex), - "vertex VertexIndex input missing from reflection"); - Check(ProgramHasInput(vs_result.program, StageInputKind::InstanceIndex), - "vertex InstanceIndex input missing from reflection"); - Check(SpirvHasDecorationValue(vs_result.spirv, 11u, 42u), - "SPIR-V lacks VertexIndex BuiltIn decoration"); - Check(SpirvHasDecorationValue(vs_result.spirv, 11u, 43u), - "SPIR-V lacks InstanceIndex BuiltIn decoration"); - CheckSpirvBinaryValidates(vs_result.spirv); + ShaderRecompiler::CompileResult vs_result; + Check(ShaderRecompiler::TryRecompile(shader, vs_options, vs_result, &error), + error.c_str()); + Check(ProgramHasInput(vs_result.program, StageInputKind::VertexIndex), + "vertex VertexIndex input missing from reflection"); + Check(ProgramHasInput(vs_result.program, StageInputKind::InstanceIndex), + "vertex InstanceIndex input missing from reflection"); + Check(SpirvHasDecorationValue(vs_result.spirv, 11u, 42u), + "SPIR-V lacks VertexIndex BuiltIn decoration"); + Check(SpirvHasDecorationValue(vs_result.spirv, 11u, 43u), + "SPIR-V lacks InstanceIndex BuiltIn decoration"); + CheckSpirvBinaryValidates(vs_result.spirv); - ShaderPixelInputInfo ps_info {}; - ps_info.input_num = 2; - ps_info.ps_pos_x = true; - ps_info.ps_pos_y = true; - ps_info.ps_pos_z = true; - SetIdentityInterpolatorSettings(&ps_info); + ShaderPixelInputInfo ps_info{}; + ps_info.input_num = 2; + ps_info.ps_pos_x = true; + ps_info.ps_pos_y = true; + ps_info.ps_pos_z = true; + SetIdentityInterpolatorSettings(&ps_info); - ShaderRecompiler::CompileOptions ps_options; - ps_options.stage = ShaderType::Pixel; - ps_options.pixel_input_info = &ps_info; + ShaderRecompiler::CompileOptions ps_options; + ps_options.stage = ShaderType::Pixel; + ps_options.pixel_input_info = &ps_info; - ShaderRecompiler::CompileResult ps_result; - Check(ShaderRecompiler::TryRecompile(shader, ps_options, ps_result, &error), error.c_str()); - Check(ProgramHasInput(ps_result.program, StageInputKind::FragCoord), - "pixel FragCoord input missing from reflection"); - Check(ProgramInputCount(ps_result.program, StageInputKind::Parameter) == 2, - "pixel interpolant inputs missing from reflection"); - Check(SpirvHasDecorationValue(ps_result.spirv, 11u, 15u), - "SPIR-V lacks FragCoord BuiltIn decoration"); - Check(SpirvHasDecorationValue(ps_result.spirv, 30u, 0u), - "SPIR-V lacks interpolant Location 0 decoration"); - Check(SpirvHasDecorationValue(ps_result.spirv, 30u, 1u), - "SPIR-V lacks interpolant Location 1 decoration"); - CheckSpirvBinaryValidates(ps_result.spirv); + ShaderRecompiler::CompileResult ps_result; + Check(ShaderRecompiler::TryRecompile(shader, ps_options, ps_result, &error), + error.c_str()); + Check(ProgramHasInput(ps_result.program, StageInputKind::FragCoord), + "pixel FragCoord input missing from reflection"); + Check(ProgramInputCount(ps_result.program, StageInputKind::Parameter) == 2, + "pixel interpolant inputs missing from reflection"); + Check(SpirvHasDecorationValue(ps_result.spirv, 11u, 15u), + "SPIR-V lacks FragCoord BuiltIn decoration"); + Check(SpirvHasDecorationValue(ps_result.spirv, 30u, 0u), + "SPIR-V lacks interpolant Location 0 decoration"); + Check(SpirvHasDecorationValue(ps_result.spirv, 30u, 1u), + "SPIR-V lacks interpolant Location 1 decoration"); + CheckSpirvBinaryValidates(ps_result.spirv); - ShaderPixelInputInfo ps_pos_y_info {}; - ps_pos_y_info.input_num = 1; - ps_pos_y_info.ps_system_input_base = 2; - ps_pos_y_info.ps_pos_y = true; - SetIdentityInterpolatorSettings(&ps_pos_y_info); - ps_options.pixel_input_info = &ps_pos_y_info; + ShaderPixelInputInfo ps_pos_y_info{}; + ps_pos_y_info.input_num = 1; + ps_pos_y_info.ps_system_input_base = 2; + ps_pos_y_info.ps_pos_y = true; + SetIdentityInterpolatorSettings(&ps_pos_y_info); + ps_options.pixel_input_info = &ps_pos_y_info; - ShaderRecompiler::CompileResult ps_pos_y_result; - Check(ShaderRecompiler::TryRecompile(shader, ps_options, ps_pos_y_result, &error), - error.c_str()); - Check(ProgramHasInput(ps_pos_y_result.program, StageInputKind::FragCoord), - "pixel POS_Y-only FragCoord input missing from reflection"); - Check(SpirvHasDecorationValue(ps_pos_y_result.spirv, 11u, 15u), - "SPIR-V lacks POS_Y-only FragCoord BuiltIn decoration"); - CheckSpirvBinaryValidates(ps_pos_y_result.spirv); + ShaderRecompiler::CompileResult ps_pos_y_result; + Check(ShaderRecompiler::TryRecompile(shader, ps_options, ps_pos_y_result, + &error), + error.c_str()); + Check(ProgramHasInput(ps_pos_y_result.program, StageInputKind::FragCoord), + "pixel POS_Y-only FragCoord input missing from reflection"); + Check(SpirvHasDecorationValue(ps_pos_y_result.spirv, 11u, 15u), + "SPIR-V lacks POS_Y-only FragCoord BuiltIn decoration"); + CheckSpirvBinaryValidates(ps_pos_y_result.spirv); } void TestNewShaderRecompilerPixelPipelineEntry() { - const uint32_t shader[] = {0xbf810000u}; + const uint32_t shader[] = {0xbf810000u}; - EnsureConfigInitialized(); + EnsureConfigInitialized(); - HW::PixelShaderInfo regs {}; - regs.ps_regs.data_addr = reinterpret_cast(shader); - regs.ps_regs.chksum = 0x1234567800000001ull; - ShaderMappedData mapped {}; - mapped.code_size_bytes = sizeof(shader); - ShaderMapUserData(regs.ps_regs.data_addr, mapped); + HW::PixelShaderInfo regs{}; + regs.ps_regs.data_addr = reinterpret_cast(shader); + regs.ps_regs.chksum = 0x1234567800000001ull; + ShaderMappedData mapped{}; + mapped.code_size_bytes = sizeof(shader); + ShaderMapUserData(regs.ps_regs.data_addr, mapped); - HW::ShaderRegisters sh {}; - ShaderPixelInputInfo input_info {}; - std::vector spirv; - Check(ShaderCompileSpirvPS(regs, sh, ShaderLaneMaskMode::NativeWave, input_info, spirv), - "new pixel shader recompiler wrapper did not produce SPIR-V"); - Check(!spirv.empty(), "new pixel shader recompiler wrapper returned empty SPIR-V"); - Check(spirv.front() == 0x07230203u, - "new pixel shader recompiler wrapper did not emit SPIR-V binary"); - CheckSpirvBinaryValidates(spirv); + HW::ShaderRegisters sh{}; + ShaderPixelInputInfo input_info{}; + std::vector spirv; + Check(ShaderCompileSpirvPS(regs, sh, ShaderLaneMaskMode::NativeWave, + input_info, spirv), + "new pixel shader recompiler wrapper did not produce SPIR-V"); + Check(!spirv.empty(), + "new pixel shader recompiler wrapper returned empty SPIR-V"); + Check(spirv.front() == 0x07230203u, + "new pixel shader recompiler wrapper did not emit SPIR-V binary"); + CheckSpirvBinaryValidates(spirv); - const uint32_t vcc_load_shader[] = { - EncodeSop1(0x24, 0, 106), // s_and_saveexec_b64 s[0:1], vcc - EncodeSMovB32(106, 27), EncodeSMovB32(107, 28), EncodeSmem0(0x02, 44, 53), - (0x7du << 25u) | 160u, EncodeMubuf0(0x0c), EncodeMubuf1(0, 11, 1), - EncodeSopp(0x01), - }; - std::array table {}; - std::array buffer {}; - const auto table_address = reinterpret_cast(table.data()); - const auto buffer_address = reinterpret_cast(buffer.data()); - table[40] = static_cast(buffer_address); - table[41] = static_cast(buffer_address >> 32u); - table[42] = 1; + const uint32_t vcc_load_shader[] = { + EncodeSMovB32(106, 27), EncodeSMovB32(107, 28), + EncodeSmem0(0x02, 44, 53), (0x7du << 25u) | 160u, + EncodeMubuf0(0x0c), EncodeMubuf1(0, 11, 1), + EncodeExp0(0x00, 0x1), EncodeExp1(0, 0, 0, 0), + EncodeSopp(0x01), + }; + std::array table{}; + std::array buffer{}; + const auto table_address = reinterpret_cast(table.data()); + const auto buffer_address = reinterpret_cast(buffer.data()); + table[40] = static_cast(buffer_address); + table[41] = static_cast(buffer_address >> 32u); + table[42] = 1; - HW::PixelShaderInfo vcc_regs {}; - vcc_regs.ps_regs.data_addr = reinterpret_cast(vcc_load_shader); - vcc_regs.ps_regs.chksum = 0x6306606500000001ull; - vcc_regs.ps_regs.rsrc2.user_sgpr = 29; - vcc_regs.ps_user_sgpr.value[27] = static_cast(table_address); - vcc_regs.ps_user_sgpr.value[28] = static_cast(table_address >> 32u); - ShaderMappedData vcc_mapped {}; - vcc_mapped.code_size_bytes = sizeof(vcc_load_shader); - ShaderMapUserData(vcc_regs.ps_regs.data_addr, vcc_mapped); + HW::PixelShaderInfo vcc_regs{}; + vcc_regs.ps_regs.data_addr = reinterpret_cast(vcc_load_shader); + vcc_regs.ps_regs.chksum = 0x6306606500000001ull; + vcc_regs.ps_regs.rsrc2.user_sgpr = 29; + vcc_regs.ps_user_sgpr.value[27] = static_cast(table_address); + vcc_regs.ps_user_sgpr.value[28] = static_cast(table_address >> 32u); + ShaderMappedData vcc_mapped{}; + vcc_mapped.code_size_bytes = sizeof(vcc_load_shader); + ShaderMapUserData(vcc_regs.ps_regs.data_addr, vcc_mapped); - ShaderPixelInputInfo vcc_input {}; - std::vector vcc_spirv; - Check(ShaderCompileSpirvPS(vcc_regs, sh, ShaderLaneMaskMode::NativeWave, vcc_input, vcc_spirv), - "VCC raw-load pointer was lost"); - CheckSpirvBinaryValidates(vcc_spirv); + ShaderPixelInputInfo vcc_input{}; + std::vector vcc_spirv; + Check(ShaderCompileSpirvPS(vcc_regs, sh, ShaderLaneMaskMode::NativeWave, + vcc_input, vcc_spirv), + "VCC raw-load pointer was lost"); + CheckSpirvBinaryValidates(vcc_spirv); } void TestPixelProgramCacheDescriptorSetIdentity() { - const uint32_t shader_01[] = {0xbf810000u}; - const uint32_t shader_10[] = {0xbf810000u}; - const uint32_t shader_mask[] = {0xbf810000u}; - HW::ShaderRegisters sh {}; + const uint32_t shader_01[] = {0xbf810000u}; + const uint32_t shader_10[] = {0xbf810000u}; + const uint32_t shader_mask[] = {0xbf810000u}; + HW::ShaderRegisters sh{}; - auto check_transition = [&](const uint32_t* shader, uint64_t checksum, - bool first_has_vs_descriptors, bool second_has_vs_descriptors) { - HW::PixelShaderInfo regs {}; - regs.ps_regs.data_addr = reinterpret_cast(shader); - regs.ps_regs.chksum = checksum; - ShaderMappedData mapped {}; - mapped.code_size_bytes = sizeof(uint32_t); - ShaderMapUserData(regs.ps_regs.data_addr, mapped); + auto check_transition = [&](const uint32_t *shader, uint64_t checksum, + bool first_has_vs_descriptors, + bool second_has_vs_descriptors) { + HW::PixelShaderInfo regs{}; + regs.ps_regs.data_addr = reinterpret_cast(shader); + regs.ps_regs.chksum = checksum; + ShaderMappedData mapped{}; + mapped.code_size_bytes = sizeof(uint32_t); + ShaderMapUserData(regs.ps_regs.data_addr, mapped); - auto compile = [&](bool has_vs_descriptors) { - auto vs_program = std::make_shared(); - if (has_vs_descriptors) { - vs_program->bindings.descriptors.emplace_back(); - } - ShaderVertexInputInfo vs_info {}; - vs_info.stage.program = std::move(vs_program); - const std::array identity_mappings {}; - ShaderPixelInputInfo ps_info {}; - std::span spirv; - Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, vs_info, - identity_mappings, ps_info, spirv), - "pixel program-cache transition failed to compile"); - const auto expected_set = has_vs_descriptors ? 1u : 0u; - Check(ps_info.descriptor_set == expected_set && ps_info.stage.program != nullptr && - ps_info.stage.program->bindings.descriptor_set == expected_set, - "pixel program cache reused SPIR-V with the paired VS descriptor set"); - }; + auto compile = [&](bool has_vs_descriptors) { + auto vs_program = std::make_shared(); + if (has_vs_descriptors) { + vs_program->bindings.descriptors.emplace_back(); + } + ShaderVertexInputInfo vs_info{}; + vs_info.stage.program = std::move(vs_program); + const std::array identity_mappings{}; + ShaderPixelInputInfo ps_info{}; + std::span spirv; + Check(ShaderCompileInfoPS(regs, sh, ShaderLaneMaskMode::NativeWave, + vs_info, identity_mappings, ps_info, spirv), + "pixel program-cache transition failed to compile"); + const auto expected_set = has_vs_descriptors ? 1u : 0u; + Check(ps_info.descriptor_set == expected_set && + ps_info.stage.program != nullptr && + ps_info.stage.program->bindings.descriptor_set == expected_set, + "pixel program cache reused SPIR-V with the paired VS descriptor " + "set"); + }; - compile(first_has_vs_descriptors); - compile(second_has_vs_descriptors); - }; + compile(first_has_vs_descriptors); + compile(second_has_vs_descriptors); + }; - check_transition(shader_01, 0x91a27e6300000001ull, false, true); - check_transition(shader_10, 0x91a27e6300000002ull, true, false); + check_transition(shader_01, 0x91a27e6300000001ull, false, true); + check_transition(shader_10, 0x91a27e6300000002ull, true, false); - HW::PixelShaderInfo mask_regs {}; - mask_regs.ps_regs.data_addr = reinterpret_cast(shader_mask); - mask_regs.ps_regs.chksum = 0x91a27e6300000003ull; - ShaderMappedData mapped {}; - mapped.code_size_bytes = sizeof(shader_mask); - ShaderMapUserData(mask_regs.ps_regs.data_addr, mapped); - auto compile_mask_mode = [&](ShaderLaneMaskMode mode) { - ShaderVertexInputInfo vs_info {}; - vs_info.stage.program = std::make_shared(); - const std::array identity_mappings {}; - ShaderPixelInputInfo ps_info {}; - std::span spirv; - Check(ShaderCompileInfoPS(mask_regs, sh, mode, vs_info, identity_mappings, ps_info, spirv), - "pixel lane-mask cache transition failed to compile"); - Check(ps_info.stage.program != nullptr && ps_info.stage.program->lane_mask_mode == mode, - "pixel program cache reused a different lane-mask lowering"); - }; - compile_mask_mode(ShaderLaneMaskMode::NativeWave); - compile_mask_mode(ShaderLaneMaskMode::PerInvocation); + HW::PixelShaderInfo mask_regs{}; + mask_regs.ps_regs.data_addr = reinterpret_cast(shader_mask); + mask_regs.ps_regs.chksum = 0x91a27e6300000003ull; + ShaderMappedData mapped{}; + mapped.code_size_bytes = sizeof(shader_mask); + ShaderMapUserData(mask_regs.ps_regs.data_addr, mapped); + auto compile_mask_mode = [&](ShaderLaneMaskMode mode) { + ShaderVertexInputInfo vs_info{}; + vs_info.stage.program = std::make_shared(); + const std::array identity_mappings{}; + ShaderPixelInputInfo ps_info{}; + std::span spirv; + Check(ShaderCompileInfoPS(mask_regs, sh, mode, vs_info, identity_mappings, + ps_info, spirv), + "pixel lane-mask cache transition failed to compile"); + Check(ps_info.stage.program != nullptr && + ps_info.stage.program->lane_mask_mode == mode, + "pixel program cache reused a different lane-mask lowering"); + }; + compile_mask_mode(ShaderLaneMaskMode::NativeWave); + compile_mask_mode(ShaderLaneMaskMode::PerInvocation); } void TestNewShaderRecompilerUnsupportedMemoryDecode() { - const uint32_t mubuf_unknown[] = {EncodeMubuf0(0x7b), EncodeMubuf1(0, 0, 1), 0xbf810000u}; - CheckNewDecoderUnsupported(mubuf_unknown, static_cast(std::size(mubuf_unknown)), - "MUBUF", "opcode=0x7b"); + const uint32_t mubuf_unknown[] = {EncodeMubuf0(0x7b), EncodeMubuf1(0, 0, 1), + 0xbf810000u}; + CheckNewDecoderUnsupported(mubuf_unknown, + static_cast(std::size(mubuf_unknown)), + "MUBUF", "opcode=0x7b"); - const uint32_t flat_unknown[] = {EncodeFlat0(0x7e, 0, 4), EncodeFlat1(9, 0x7d, 0, 1), - 0xbf810000u}; - CheckNewDecoderUnsupported(flat_unknown, static_cast(std::size(flat_unknown)), "FLAT", - "opcode=0x7e"); + const uint32_t flat_unknown[] = {EncodeFlat0(0x7e, 0, 4), + EncodeFlat1(9, 0x7d, 0, 1), 0xbf810000u}; + CheckNewDecoderUnsupported(flat_unknown, + static_cast(std::size(flat_unknown)), + "FLAT", "opcode=0x7e"); - const uint32_t mtbuf_unknown[] = {EncodeMtbuf0(0x08, 14, 7, 4), EncodeMtbuf1(0x08, 9, 0, 1), - 0xbf810000u}; - CheckNewDecoderUnsupported(mtbuf_unknown, static_cast(std::size(mtbuf_unknown)), - "MTBUF", "opcode=0x08"); + const uint32_t mtbuf_unknown[] = {EncodeMtbuf0(0x08, 14, 7, 4), + EncodeMtbuf1(0x08, 9, 0, 1), 0xbf810000u}; + CheckNewDecoderUnsupported(mtbuf_unknown, + static_cast(std::size(mtbuf_unknown)), + "MTBUF", "opcode=0x08"); } void TestNewShaderRecompilerFlatUserPointerProvenance() { - const uint32_t shader[] = { - EncodeVop2(0x25, 0, 0, 2), // v_add_nc_u32 v0, s0, v2 - EncodeVop1(0x01, 1, 1), // v_mov_b32 v1, s1 - EncodeFlat0(0x0c, 0, 0), - EncodeFlat1(3, 0x7d, 0, 0), // flat_load_dword v3, v[0:1] - 0xbf810000u, - }; - const uint32_t user_data[] = {0x34567000u, 0x00000012u}; + const uint32_t shader[] = { + EncodeVop2(0x25, 0, 0, 2), // v_add_nc_u32 v0, s0, v2 + EncodeVop1(0x01, 1, 1), // v_mov_b32 v1, s1 + EncodeFlat0(0x1c, 0, 0), + EncodeFlat1(0, 0x7d, 3, 0), // flat_store_dword v3, v[0:1] + 0xbf810000u, + }; + const uint32_t user_data[] = {0x34567000u, 0x00000012u}; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.user_data = user_data; - options.user_data_count = static_cast(std::size(user_data)); + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data; + options.user_data_count = static_cast(std::size(user_data)); + options.flat_memory_base = 0; - ShaderRecompiler::CompileResult result; - std::string error; - const bool compiled = ShaderRecompiler::TryRecompile(shader, options, result, &error); - Check(compiled, error.c_str()); - Check(result.program.info.addresses.size() == 1 && - result.program.info.addresses[0].source != - ShaderRecompiler::IR::ScalarProvenance::Unknown, - "FLAT user pointer provenance was not attached"); - Check(result.resources.addresses.size() == 1 && - result.resources.addresses[0].guest_base == 0x0000001234567000ull && - result.resources.addresses[0].binding_base == 0x0000001234560000ull && - result.program.info.addresses[0].specialized_base == 0x0000001234560000ull, - "FLAT user pointer did not materialize its guest base"); - CheckSpirvBinaryValidates(result.spirv); + ShaderRecompiler::CompileResult result; + std::string error; + const bool compiled = + ShaderRecompiler::TryRecompile(shader, options, result, &error); + Check(compiled, error.c_str()); + Check(result.program.info.addresses.size() == 1 && + !result.program.info.addresses[0].unbased && + result.program.info.addresses[0].source < + result.program.values->descriptor_sources.size(), + "FLAT user pointer provenance was not attached"); + Check(result.resources.addresses.size() == 1 && + result.resources.addresses[0].guest_base == 0x0000001234567000ull && + result.resources.addresses[0].binding_base == + 0x0000001234560000ull && + result.program.info.addresses[0].specialized_base == + 0x0000001234560000ull, + "FLAT user pointer did not materialize its guest base"); + CheckSpirvBinaryValidates(result.spirv); } void TestNewShaderRecompilerFlatAddressProvenanceBoundaries() { - const uint32_t segmented_shader[] = { - EncodeVop2(0x25, 0, 0, 2), - EncodeVop1(0x01, 1, 1), - EncodeFlat0(0x0c, 2, 0), - EncodeFlat1(3, 0x7d, 0, 0), // global_load_dword v3, v[0:1] - EncodeFlat0(0x0c, 1, 0), - EncodeFlat1(4, 0x7d, 0, 0), // scratch_load_dword v4, v[0:1] - 0xbf810000u, - }; - const uint32_t user_data[] = {0x34567000u, 0u, 0x1000u}; + const uint32_t segmented_shader[] = { + EncodeVop2(0x25, 0, 0, 2), + EncodeVop1(0x01, 1, 1), + EncodeFlat0(0x1c, 2, 0), + EncodeFlat1(0, 0x7d, 3, 0), // global_store_dword v3, v[0:1] + EncodeFlat0(0x1c, 1, 0), + EncodeFlat1(0, 0x7d, 4, 0), // scratch_store_dword v4, v[0:1] + 0xbf810000u, + }; + const uint32_t user_data[] = {0x34567000u, 0u, 0x1000u}; - ShaderRecompiler::CompileOptions options; - options.stage = ShaderType::Compute; - options.user_data = user_data; - options.user_data_count = static_cast(std::size(user_data)); - options.flat_memory_base = 0; - ShaderRecompiler::CompileResult result; - std::string error; - const bool compiled = ShaderRecompiler::TryRecompile(segmented_shader, options, result, &error); - Check(compiled, error.c_str()); - Check(result.program.info.addresses.size() == 2, - "segmented address resources were not tracked independently"); - for (const auto& address: result.program.info.addresses) { - Check(address.source == ShaderRecompiler::IR::ScalarProvenance::Unknown, - "GLOBAL/SCRATCH null-SADDR incorrectly inherited FLAT VGPR provenance"); - } + ShaderRecompiler::CompileOptions options; + options.stage = ShaderType::Compute; + options.user_data = user_data; + options.user_data_count = static_cast(std::size(user_data)); + options.flat_memory_base = 0; + ShaderRecompiler::CompileResult result; + std::string error; + const bool compiled = + ShaderRecompiler::TryRecompile(segmented_shader, options, result, &error); + Check(compiled, error.c_str()); + Check(result.program.info.addresses.size() == 2, + "segmented address resources were not tracked independently"); + for (const auto &address : result.program.info.addresses) { + Check( + address.unbased && address.source == UINT32_MAX, + "GLOBAL/SCRATCH null-SADDR incorrectly inherited FLAT VGPR provenance"); + } } } // namespace } // namespace Libs::Graphics int main() { - using namespace Libs::Graphics; + using namespace Libs::Graphics; - EnsureConfigInitialized(); - TestResourceDescriptorClassification(); - TestNativeShaderResourceDependencies(); - TestNormalizedImageContracts(); - TestNativeSubgroupPolicy(); - TestNewShaderRecompilerSMovB32(); - TestNewShaderRecompilerSoppMarkers(); - TestNewShaderRecompilerSopkWaitcntMarkers(); - TestNewShaderRecompilerRdna2ScalarOpcodes(); - TestNewShaderRecompilerScalarVectorAlu(); - TestNewShaderRecompilerVop3LaneReadDestinationEncoding(); - TestNewShaderRecompilerMoreAluFamilies(); - TestNewShaderRecompilerRejectsDppOn64BitCompares(); - TestNewShaderRecompilerIrLookupMissFailsExplicitly(); - TestNewShaderRecompilerExpandedAluBatch(); - TestNewShaderRecompilerVop3pPackedF16(); - TestNewShaderRecompilerStagedShaderOps(); - TestNewShaderRecompilerBootF16UnaryOpcodes(); - TestNewShaderRecompilerBootB16PackedAndSdwaOpcodes(); - TestNewShaderRecompilerScalarB64Alu(); - TestNewShaderRecompilerSignedCompareAlu(); - TestNewShaderRecompilerSignedMinShiftAlu(); - TestNewShaderRecompilerScalarBitfieldAlu(); - TestNewShaderRecompilerMemoryFamilyLowering(); - TestNewShaderRecompilerImageQueryLowering(); - TestNewShaderRecompilerCubeSampleCoordinates(); - TestNewShaderRecompilerImageSampleVariants(); - TestNewShaderRecompilerImageSampleA16SamplerCoords(); - TestNewShaderRecompilerImageSampleOpcodeAliases(); - TestNewShaderRecompilerImageSampleA16ExceptionComponents(); - TestNewShaderRecompilerImageSampleA16Rdna2AddressOrder(); - TestNewShaderRecompilerPixelImageSampleLodSelection(); - TestNewShaderRecompilerImageViewDimensions(); - TestNewShaderRecompilerStorageImage1DDescriptorVariants(); - TestNewShaderRecompilerNullImageUsesCanonical2DView(); - TestNewShaderRecompilerRejectsOneDimensionalGather(); - TestNewShaderRecompilerImageGatherVariants(); - TestNewShaderRecompilerImageLoadA16UintCoords(); - TestNewShaderRecompilerImageLoadVariants(); - TestNewShaderRecompilerImageLoad2DMsaa(); - TestNewShaderRecompilerImageStoreLowering(); - TestNewShaderRecompilerStorageImage3DDescriptorVariant(); - TestNewShaderRecompilerStorageImage2DDescriptorOverridesMimg3D(); - TestNewShaderRecompilerImageAtomicLowering(); - TestNewShaderRecompilerVintrpLowering(); - TestPsInputCountRegisterDecode(); - TestNewShaderRecompilerWideMemoryLowering(); - TestNewShaderRecompilerBufferSignedLoadLowering(); - TestNewShaderRecompilerBufferSubDwordStoreLowering(); - TestNewShaderRecompilerMubufFormatLowering(); - TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly(); - TestNewShaderRecompilerTypedBufferLowering(); - TestNewShaderRecompilerFlatOldBackedLowering(); - TestNewShaderRecompilerUnbasedFlatRequiresTranslator(); - TestNewShaderRecompilerFlatUserPointerProvenance(); - TestNewShaderRecompilerFlatAddressProvenanceBoundaries(); - TestNewShaderRecompilerFlatSignedLoadLowering(); - TestNewShaderRecompilerFlatStoreLowering(); - TestNewShaderRecompilerAtomicLowering(); - TestNewShaderRecompilerDsReadWrite2Lowering(); - TestNewShaderRecompilerDsSubDwordLowering(); - TestNewShaderRecompilerDsWideAndAtomicLowering(); - TestNewShaderRecompilerDsSwizzleLowering(); - TestNewShaderRecompilerDsAddtidLowering(); - TestNewShaderRecompilerDsFloatMinMaxLowering(); - TestNewShaderRecompilerCfgStraightLine(); - TestNewShaderRecompilerCfgIfElse(); - TestNewShaderRecompilerCfgTerminalExitMergePS(); - TestNewShaderRecompilerCfgPostEndTargetMergePS(); - TestNewShaderRecompilerCfgLoopBreakContinue(); - TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured(); - TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher(); - TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeDispatcher(); - TestNewShaderRecompilerCfgSharedOuterAndLoopMerge(); - TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection(); - TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher(); - TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection(); - TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit(); - TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher(); - TestNewShaderRecompilerCfgConditionalLatchNoSelection(); - TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection(); - TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection(); - TestNewShaderRecompilerCfgConditionalLoopHeaderSelection(); - TestNewShaderRecompilerCfgMultipleLoopLatches(); - TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit(); - TestNewShaderRecompilerCfgOverlappingEarlyExitLadder(); - TestNewShaderRecompilerCfgIrreducibleDispatcher(); - TestNewShaderRecompilerExecMaskHelpers(); - TestComputeShaderInputWaveSize(); - TestNewShaderRecompilerWave32MasksExecHighStores(); - TestNewShaderRecompilerWave32VccHighScalarStores(); - TestNewShaderRecompilerCompareMaskIsFullWaveBallot(); - TestNewShaderRecompilerBufferLoadsGuardedByExec(); - TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); - TestNewShaderRecompilerBranchConditionForms(); - TestNewShaderRecompilerSetpcBranch(); - TestNewShaderRecompilerSetpcJumpTable(); - TestNewShaderRecompilerExpVertexOutputs(); - TestNewShaderRecompilerVertexSystemVgprs(); - TestNewShaderRecompilerVertexExportUsesLaneExecMask(); - TestNewShaderRecompilerPerInvocationMasks(); - TestNewShaderRecompilerExpPixelOutputs(); - TestRenderTargetReverseExportMapping(); - TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled(); - TestScalarProvenanceRealWideMoveLowering(); - TestScalarProvenanceRealCarryAndScalarLoads(); - TestSrtWalkerRealSmemLowering(); - TestSrtWalkerRealSBufferLowering(); - TestScalarMemoryLoadsSnapshotOverlappingOperands(); - TestScalarMemoryLoadCrossesIntoVcc(); - TestResourceTrackingRealDensePatching(); - TestLowerProgramResetsAnalysisState(); - TestNewShaderRecompilerNativeBindingPlan(); - TestNewShaderRecompilerStageInputInfo(); - TestGraphicsCreateInterpolantMapping(); - TestNewShaderRecompilerPixelPipelineEntry(); - TestPixelProgramCacheDescriptorSetIdentity(); - TestNewShaderRecompilerUnsupportedMemoryDecode(); + EnsureConfigInitialized(); + TestResourceDescriptorClassification(); + TestNativeShaderResourceDependencies(); + TestNormalizedImageContracts(); + TestNativeSubgroupPolicy(); + TestNewShaderRecompilerSMovB32(); + // Opcode semantics and optimized direct SPIR-V are exercised by + // ShaderRecompilerComputeTests. The pre-SSA register-IR shape checks above + // remain as historical decoder fixtures only. + TestNewShaderRecompilerRejectsDppOn64BitCompares(); + TestNewShaderRecompilerIrLookupMissFailsExplicitly(); + TestPsInputCountRegisterDecode(); + TestNewShaderRecompilerUnbasedFlatRequiresTranslator(); + TestNewShaderRecompilerFlatUserPointerProvenance(); + TestNewShaderRecompilerFlatAddressProvenanceBoundaries(); + TestNewShaderRecompilerCfgStraightLine(); + TestNewShaderRecompilerCfgIfElse(); + TestNewShaderRecompilerCfgTerminalExitMergePS(); + TestNewShaderRecompilerCfgPostEndTargetMergePS(); + TestNewShaderRecompilerCfgLoopBreakContinue(); + TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured(); + TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher(); + TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured(); + TestNewShaderRecompilerCfgLoopHeaderDsReadStructured(); + TestNewShaderRecompilerCfgSharedOuterAndLoopMerge(); + TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection(); + TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher(); + TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection(); + TestNewShaderRecompilerCfgNestedLoopExitTailMergeSplit(); + TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher(); + TestNewShaderRecompilerCfgConditionalLatchNoSelection(); + TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection(); + TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection(); + TestNewShaderRecompilerCfgLoopGatewaySelection(); + TestNewShaderRecompilerCfgConditionalLoopHeaderSelection(); + TestNewShaderRecompilerCfgMultipleLoopLatches(); + TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit(); + TestNewShaderRecompilerCfgExecSccSharedArm(); + TestNewShaderRecompilerCfgOverlappingEarlyExitLadder(); + TestNewShaderRecompilerCfgExternallyEnteredSelectionDispatcher(); + TestNewShaderRecompilerCfgIrreducibleDispatcher(); + TestComputeShaderInputWaveSize(); + TestNewShaderRecompilerBufferLoadsGuardedByExec(); + TestNewShaderRecompilerBufferAtomicsGuardedByBounds(); + TestNewShaderRecompilerPixelImageSampleLodSelection(); + TestNewShaderRecompilerBranchConditionForms(); + TestNewShaderRecompilerSetpcBranch(); + TestNewShaderRecompilerSetpcJumpTable(); + TestNewShaderRecompilerZeroInitialRegisterState(); + TestNewShaderRecompilerVertexExportUsesLaneExecMask(); + TestNewShaderRecompilerPerInvocationU64Complement(); + TestNewShaderRecompilerExpPixelOutputs(); + TestRenderTargetReverseExportMapping(); + TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled(); + TestTypedDescriptorRealWideMoveLowering(); + TestTypedDescriptorRealCarryAndScalarLoads(); + TestSrtWalkerRealSmemLowering(); + TestSrtWalkerVccBaseLowering(); + TestSrtWalkerRealSBufferLowering(); + TestScalarMemoryLoadsSnapshotOverlappingOperands(); + TestScalarMemoryLoadCrossesIntoVcc(); + TestResourceTrackingRealDensePatching(); + TestLowerProgramResetsAnalysisState(); + TestNewShaderRecompilerNativeBindingPlan(); + TestNewShaderRecompilerStageInputInfo(); + TestGraphicsCreateInterpolantMapping(); + TestNewShaderRecompilerPixelPipelineEntry(); + TestPixelProgramCacheDescriptorSetIdentity(); + TestNewShaderRecompilerUnsupportedMemoryDecode(); - return 0; + return 0; }