From 6799ecbc5be06680444c0fb34f305c73f332e55d Mon Sep 17 00:00:00 2001 From: nmzik Date: Tue, 29 Sep 2026 18:41:22 +0400 Subject: [PATCH] shader: add native 64-bit LDS OR with shared typed storage views --- .../presentation/window/vulkanWindow.cpp | 27 +++- .../recompiler/backend/spirv/SpirvEmitter.cpp | 6 + .../backend/spirv/spirvEmitterInstructions.h | 1 + .../backend/spirv/spirvEmitterInternal.h | 3 + .../backend/spirv/spirvEmitterMemory.cpp | 27 ++++ .../spirv/spirvEmitterMemoryHelpers.cpp | 31 ++++- .../backend/spirv/spirvEmitterModule.cpp | 7 +- .../recompiler/frontend/decode/MemoryOps.cpp | 2 + .../frontend/decode/ShaderDecoder.cpp | 1 + .../frontend/decode/ShaderDecoder.h | 1 + .../recompiler/frontend/translate/Memory.cpp | 7 +- .../recompiler/ir/opcodes/ValueOpcodes.cpp | 2 + .../recompiler/ir/opcodes/ValueOpcodes.inc | 1 + tests/ShaderRecompilerComputeTests.cpp | 121 +++++++++++++++++- 14 files changed, 221 insertions(+), 16 deletions(-) diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 15a2e1016..824943eaa 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -520,16 +520,20 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, #endif depth_clip_control.depthClipControl = VK_TRUE; - auto features12 = WindowContext::RequiredVulkan12Features(); - features12.pNext = &depth_clip_control; - + const bool workgroup_layout_extension = + HasExtension(device_extensions, VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME); + vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR supported_workgroup_layout {}; + vk::PhysicalDeviceVulkan12Features supported_features12 {}; + supported_features12.pNext = workgroup_layout_extension ? &supported_workgroup_layout : nullptr; vk::PhysicalDeviceVulkan13Features supported_features13 {}; + supported_features13.pNext = &supported_features12; const auto robustness2_ext_enabled = HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); vk::PhysicalDeviceRobustness2FeaturesEXT supported_robustness2 {}; if (robustness2_ext_enabled) { + supported_robustness2.pNext = supported_features13.pNext; supported_features13.pNext = &supported_robustness2; } @@ -557,6 +561,22 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, supported_features2.pNext = &provoking_vertex; } physical_device.getFeatures2(&supported_features2); + + auto features12 = WindowContext::RequiredVulkan12Features(); + features12.shaderSharedInt64Atomics = supported_features12.shaderSharedInt64Atomics; + vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout {}; + workgroup_layout.workgroupMemoryExplicitLayout = + supported_workgroup_layout.workgroupMemoryExplicitLayout; + workgroup_layout.pNext = &depth_clip_control; + features12.pNext = workgroup_layout_extension ? static_cast(&workgroup_layout) + : static_cast(&depth_clip_control); + if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) { + Log::WriteToConsoleAndLog(fmt::format( + "WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, " + "workgroupMemoryExplicitLayout={}. Continuing with the selected Vulkan device.\n", + features12.shaderSharedInt64Atomics != VK_FALSE, + workgroup_layout.workgroupMemoryExplicitLayout != VK_FALSE)); + } graphics.mesh_shader_enabled = mesh_extension && supported_mesh.meshShader; vk::PhysicalDeviceSubgroupSizeControlProperties subgroup_size_control {}; @@ -1049,6 +1069,7 @@ void WindowContext::CreateVulkan() { for (const auto* extension: {VK_EXT_ROBUSTNESS_2_EXTENSION_NAME, VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME, VK_EXT_MESH_SHADER_EXTENSION_NAME, + VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, VK_EXT_DEPTH_RANGE_UNRESTRICTED_EXTENSION_NAME}) { if (HasExtension(available_extensions, extension)) { device_extensions.push_back(extension); diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index c45a0c9d0..ee6d56444 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -229,6 +229,12 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { Fail(program, "shared operation has invalid resource kind"); } + if (inst.GetOpcode() == IR::ValueOpcode::SharedAtomicOr64) { + if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) { + Fail(program, "64-bit shared atomics require compute LDS"); + } + requirements.shared_int64_atomics = true; + } if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh && kind == IR::ResourceKind::Lds) { requirements.function_lds = true; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index d12020db2..b835b5a53 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -222,6 +222,7 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst void EmitStoreMemory(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitAtomic32(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst); +void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst); uint32_t EmitSharedIncDec(ValueEmitContext& ctx, const IR::Inst& inst); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 541632323..17d68a6a7 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -67,6 +67,7 @@ struct SpirvRequirements { bool function_scratch = false; bool pixel_valid_mask = false; bool buffer_int64_atomics = false; + bool shared_int64_atomics = false; bool coherent_buffers = false; bool float64 = false; }; @@ -101,6 +102,7 @@ struct EmitterState { uint32_t shader_data_storage_variable = 0; uint32_t flattened_srt_variable = 0; uint32_t lds_variable = 0; + uint32_t lds_u64_variable = 0; std::array scratch_variable {}; std::array image_variables {}; uint32_t sampler_variable = 0; @@ -376,6 +378,7 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR:: void EmitMemoryOffsets(EmitterState& state); uint32_t LdsDwordCount(const EmitterState& state); +void EnsureLdsStorage(EmitterState& state); struct MemoryResourceAccess { IR::ResourceKind kind = IR::ResourceKind::None; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index d16cf087a..741d94283 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -548,6 +548,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd; case IR::ValueOpcode::BufferAtomicOr32: case IR::ValueOpcode::BufferAtomicOr64: + case IR::ValueOpcode::SharedAtomicOr64: case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr; case IR::ValueOpcode::BufferAtomicXor32: case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor; @@ -1064,6 +1065,32 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { }); } +void EmitSharedAtomicOr64(ValueEmitContext& ctx, const IR::Inst& inst) { + auto& state = ctx.state; + const auto& mem = ctx.Memory(inst); + EnsureLdsStorage(state); + EmitIfCondition(state, ctx.Arg(inst, 2), [&]() { + const auto address = Binary(state, spv::OpBitwiseAnd, TypeU32(state), + ByteAddress(ctx, inst, mem), ConstantU32(state, 0xfff8u)); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, + ConstantU32(state, 3u)); + const auto in_bounds = Binary(state, spv::OpULessThan, TypeBool(state), index, + ConstantU32(state, LdsDwordCount(state) / 2u)); + EmitIfCondition(state, in_bounds, [&]() { + const auto pointer = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpAccessChain, + TypePointer(state, spv::StorageClassWorkgroup, TypeScalarU64(state)), + pointer, state.lds_u64_variable, ConstantU32(state, 0), index); + const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, 1)); + state.builder.AddFunction( + SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), state.builder.AllocateId(), + pointer, ConstantU32(state, spv::ScopeWorkgroup), + ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | + spv::MemorySemanticsWorkgroupMemoryMask), value); + }); + }); +} + uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) { const auto& mem = ctx.Memory(inst); const bool max_value = inst.GetOpcode() == IR::ValueOpcode::BufferAtomicFMax32; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 2aee17430..0ccad9a62 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -63,16 +63,34 @@ uint32_t LdsDwordCount(const EmitterState& state) { return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u; } -static void EnsureLdsStorage(EmitterState& state) { +void EnsureLdsStorage(EmitterState& state) { if (state.lds_variable != 0) { return; } if (ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) { EXIT("function LDS was not prepared before SPIR-V function emission\n"); } - state.lds_variable = state.builder.DefineGlobalVariable( - TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsDwordCount(state)), - spv::StorageClassWorkgroup); + const auto define = [&](uint32_t type, uint32_t bytes) { + const auto array = state.builder.DecoratedType( + spv::OpTypeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, bytes}}}, type, + ConstantU32(state, std::max(LdsDwordCount(state) * 4u / bytes, 1u))); + const auto block = state.builder.DecoratedType( + spv::OpTypeStruct, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, + {spv::OpDecorate, {spv::DecorationBlock}}}, array); + const auto variable = state.builder.DefineGlobalVariable( + TypePointer(state, spv::StorageClassWorkgroup, block), spv::StorageClassWorkgroup); + state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); + return variable; + }; + if (state.requirements.shared_int64_atomics) { + state.lds_variable = define(TypeU32(state), 4u); + state.lds_u64_variable = define(TypeScalarU64(state), 8u); + state.builder.AddName(state.lds_u64_variable, "lds_qwords"); + } else { + state.lds_variable = state.builder.DefineGlobalVariable( + TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsDwordCount(state)), + spv::StorageClassWorkgroup); + } state.builder.AddName(state.lds_variable, "lds_dwords"); } @@ -167,6 +185,11 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces : ShaderWorkgroupInput(state.program.stage, state.input_info) != nullptr ? spv::StorageClassWorkgroup : spv::StorageClassFunction; + if (access.kind == IR::ResourceKind::Lds && state.requirements.shared_int64_atomics) { + state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class), + pointer, access.object_pointer, ConstantU32(state, 0), index); + return pointer; + } state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class), pointer, access.object_pointer, index); return pointer; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 90d332613..e693a999f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -626,10 +626,15 @@ void DefineModule(EmitterState& state) { state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses); state.builder.RequireExtension("SPV_KHR_physical_storage_buffer"); } - if (state.requirements.buffer_int64_atomics) { + if (state.requirements.buffer_int64_atomics || state.requirements.shared_int64_atomics) { state.builder.RequireCapability(spv::CapabilityInt64); state.builder.RequireCapability(spv::CapabilityInt64Atomics); } + if (state.requirements.shared_int64_atomics) { + state.builder.RequireVersion(0x00010400u); + state.builder.RequireExtension("SPV_KHR_workgroup_memory_explicit_layout"); + state.builder.RequireCapability(spv::CapabilityWorkgroupMemoryExplicitLayoutKHR); + } if (state.clip_distance_variable != 0) { state.builder.RequireCapability(spv::CapabilityClipDistance); } diff --git a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp index 90102824d..727646e66 100644 --- a/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp @@ -106,6 +106,7 @@ constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = { {0x39u, Opcode::DS_READ_I8, 1, 8, true}, {0x3au, Opcode::DS_READ_U8, 1, 8}, {0x3bu, Opcode::DS_READ_I16, 1, 16, true}, {0x3cu, Opcode::DS_READ_U16, 1, 16}, {0x3du, Opcode::DS_CONSUME, 1, 32}, {0x3eu, Opcode::DS_APPEND, 1, 32}, + {0x4au, Opcode::DS_OR_B64, 2, 32}, {0x4du, Opcode::DS_WRITE_B64, 2, 32}, {0x4eu, Opcode::DS_WRITE2_B64, 4, 32}, {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32}, {0x76u, Opcode::DS_READ_B64, 2, 32}, {0x77u, Opcode::DS_READ2_B64, 4, 32}, {0x78u, Opcode::DS_READ2ST64_B64, 4, 32}, @@ -185,6 +186,7 @@ bool IsDsAtomicOpcode(Opcode opcode) { case Opcode::DS_AND_B32: case Opcode::DS_AND_RTN_B32: case Opcode::DS_OR_B32: + case Opcode::DS_OR_B64: case Opcode::DS_OR_RTN_B32: case Opcode::DS_XOR_B32: case Opcode::DS_XOR_RTN_B32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp index 36b53e443..04ce1ff35 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp @@ -681,6 +681,7 @@ std::string InstructionToString(const Instruction& inst) { case Opcode::DS_AND_B32: case Opcode::DS_AND_RTN_B32: case Opcode::DS_OR_B32: + case Opcode::DS_OR_B64: case Opcode::DS_OR_RTN_B32: case Opcode::DS_XOR_B32: case Opcode::DS_XOR_RTN_B32: diff --git a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h index e23b52041..f3024657e 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h +++ b/src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h @@ -528,6 +528,7 @@ enum class Opcode { DS_AND_B32, DS_AND_RTN_B32, DS_OR_B32, + DS_OR_B64, DS_OR_RTN_B32, DS_XOR_B32, DS_XOR_RTN_B32, diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index db7de4002..28fb49ddb 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -570,7 +570,10 @@ void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opc bool returns_value) { const auto memory = MemoryInfoFromDecoded(inst); const auto address = ReadU32(MemorySourceAt(inst, 1)); - const auto result = ir.Emit(opcode, {address, ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()}, + const auto data_src = MemorySourceAt(inst, 0); + const IR::Value value = + memory.data_dwords == 2u ? IR::Value(ReadU64(data_src)) : IR::Value(ReadU32(data_src)); + const auto result = ir.Emit(opcode, {address, value, ir.GetExec()}, AddMemoryInfo(memory, inst.pc)); if (returns_value) { WriteOperand(inst.dst, result); @@ -1003,6 +1006,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicAnd32, true); case Decoder::Opcode::DS_OR_B32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicOr32, false); + case Decoder::Opcode::DS_OR_B64: + return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicOr64, false); case Decoder::Opcode::DS_OR_RTN_B32: return DS_ATOMIC(inst, IR::ValueOpcode::SharedAtomicOr32, true); case Decoder::Opcode::DS_XOR_B32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index d18873871..184509ccf 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -173,6 +173,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) { case ValueOpcode::SharedAtomicUMax32: case ValueOpcode::SharedAtomicAnd32: case ValueOpcode::SharedAtomicOr32: + case ValueOpcode::SharedAtomicOr64: case ValueOpcode::SharedAtomicXor32: return SharedAccess::Atomic; case ValueOpcode::DataAppend: return SharedAccess::Append; case ValueOpcode::DataConsume: return SharedAccess::Consume; @@ -182,6 +183,7 @@ SharedAccess SharedAccessOf(ValueOpcode opcode) { uint32_t SharedComponentCount(ValueOpcode opcode) { switch (opcode) { + case ValueOpcode::SharedAtomicOr64: case ValueOpcode::LoadSharedU32x2: case ValueOpcode::WriteSharedU32x2: return 2u; case ValueOpcode::LoadSharedU32x3: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 1941e812c..b6d4a8c52 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -276,6 +276,7 @@ VALUE_OPCODE(SharedAtomicSMax32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1) VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1) +VALUE_OPCODE(SharedAtomicOr64, Void, U32, U64, U1) VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1) VALUE_OPCODE(DataAppend, U32, U32, U1) VALUE_OPCODE(DataConsume, U32, U32, U1) diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 8edeeb01f..250293511 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -16198,6 +16198,8 @@ private: (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) { vk::PhysicalDeviceVulkan12Features features12{}; features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; + vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{}; + features12.pNext = &workgroup_layout; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR; barycentric.pNext = &features12; @@ -16207,6 +16209,11 @@ private: physical.getFeatures2(&features); if (barycentric.fragmentShaderBarycentric != true || features.features.shaderInt64 != true || + features12.samplerMirrorClampToEdge != true || + features12.shaderOutputViewportIndex != true || + features12.shaderBufferInt64Atomics != true || + features12.shaderSharedInt64Atomics != true || + workgroup_layout.workgroupMemoryExplicitLayout != true || features12.bufferDeviceAddress != true) { continue; } @@ -16220,7 +16227,7 @@ private: } } Require("VulkanHarness", "dispatch", m_physical_device != nullptr, - "no Vulkan graphics+compute device with fragment barycentrics"); + "no Vulkan graphics+compute device with fragment barycentrics and 64-bit LDS atomics"); m_physical_device.getMemoryProperties(&m_memory_properties); vk::PhysicalDeviceFeatures available_features{}; @@ -16305,12 +16312,11 @@ private: device_info.sType = vk::StructureType::eDeviceCreateInfo; device_info.queueCreateInfoCount = 1; device_info.pQueueCreateInfos = &queue_info; - vk::PhysicalDeviceVulkan12Features device_features12{}; - device_features12.sType = - vk::StructureType::ePhysicalDeviceVulkan12Features; - device_features12.timelineSemaphore = true; - device_features12.bufferDeviceAddress = true; - device_features12.shaderOutputLayer = true; + auto device_features12 = WindowContext::RequiredVulkan12Features(); + device_features12.shaderSharedInt64Atomics = true; + vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{}; + workgroup_layout.workgroupMemoryExplicitLayout = true; + device_features12.pNext = &workgroup_layout; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR; barycentric.pNext = &device_features12; @@ -16365,6 +16371,7 @@ private: VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME, VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME, + VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME}; if (m_rasterization_supported) { device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME); @@ -26858,6 +26865,95 @@ TestCase DsAtomicReturnVariants() { O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase DsOrB64Bounds() { + using O = ShaderOpcode; + TestCase test; + test.name = "DsOrB64Bounds"; + auto &code = test.code; + AppendVMovU32(&code, 0, 0x1234); + AppendVMovU32(&code, 7, 0); + const u32 initial[] = {0x100, 0x200, 0x400, 0x800, 0x13579bdf}; + for (u32 i = 0; i < std::size(initial); ++i) { + AppendVMovLiteral(&code, 3, initial[i]); + code.push_back(EncodeDs0(0x0d, i * 4u)); + code.push_back(EncodeDs1(0, 3, 7)); + } + AppendVMovU32(&code, 3, 1); + AppendVMovU32(&code, 4, 2); + AppendVMovU32(&code, 7, 3); // The 64-bit address ignores the low three bits. + code.insert(code.end(), {0xd9280000u, 0x00000307u}); + AppendVMovLiteral(&code, 7, 0x10007); + code.push_back(EncodeDs0(0x4a, 2)); // Offset before 16-bit wrap/alignment -> 8. + code.push_back(EncodeDs1(0, 3, 7)); + AppendVMovLiteral(&code, 3, 0xffffffffu); + AppendVMovLiteral(&code, 4, 0xffffffffu); + AppendVMovU32(&code, 7, 16); // Only one DWORD remains: ignore the entire OR. + code.insert(code.end(), {0xd9280000u, 0x00000307u}); + AppendVMovU32(&code, 7, 0); + code.push_back(EncodeSMovB32(126, InlineU32(0))); + code.insert(code.end(), {0xd9280000u, 0x00000307u}); + code.push_back(EncodeSMovB32(126, InlineU32(1))); + for (u32 i = 0; i < std::size(initial); ++i) { + code.push_back(EncodeDs0(0x36, i * 4u)); + code.push_back(EncodeDs1(8, 0, 7)); + AppendStoreVgpr(&code, 8, i); + } + AppendStoreVgpr(&code, 0, 5); // No-return DS must not overwrite the encoded VDst. + AppendEnd(&code); + test.expected = {0x101, 0x202, 0x401, 0x802, 0x13579bdf, 0x1234}; + test.compute_info.lds_size_dwords = 5; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::DS_WRITE_B32, O::DS_OR_B64, + O::DS_READ_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicOr %ulong", "WorkgroupMemoryExplicitLayoutKHR", "Aliased"}; + return test; +} + +TestCase DsOrB64Contention(u32 wave_size) { + using O = ShaderOpcode; + TestCase test; + test.name = wave_size == 32 ? "DsOrB64ContentionWave32" : "DsOrB64ContentionWave64"; + auto &code = test.code; + AppendVMovU32(&code, 7, 0); + code.push_back(EncodeSop1(0x04, 10, 126)); + code.push_back(EncodeVopc(0xc2, InlineU32(0), 0)); + code.push_back(EncodeSop1(0x04, 126, 106)); + for (u32 offset : {0, 4}) { + code.push_back(EncodeDs0(0x0d, offset)); + code.push_back(EncodeDs1(0, 7, 7)); + } + code.push_back(EncodeSop1(0x04, 126, 10)); + code.push_back(EncodeSopp(0x0a, 0)); + AppendVMovU32(&code, 5, 1); + AppendVMovU32(&code, 6, 2); + code.push_back(EncodeVop2(0x1a, 3, Vgpr(0), 5)); + code.push_back(EncodeVop2(0x1a, 4, Vgpr(0), 6)); + AppendSMovLiteral(&code, 126, 0x55555555u); + AppendSMovLiteral(&code, 127, 0xaaaaaaaau); + code.insert(code.end(), {0xd9280000u, 0x00000307u}); + code.push_back(EncodeSop1(0x04, 126, 10)); + code.push_back(EncodeSopp(0x0a, 0)); + for (u32 half = 0; half < 2; ++half) { + code.push_back(EncodeDs0(0x36, half * 4)); + code.push_back(EncodeDs1(8, 0, 7)); + AppendStoreVgprAtLaneDwordOffset(&code, 8, 0, half * 128); + } + AppendEnd(&code); + test.expected.assign(128, wave_size == 32 ? 0x55555555u : 0xffffffffu); + test.expected.resize(256, wave_size == 32 ? 0xaaaaaaaau : 0xfffffffeu); + test.compute_info.threads_num[0] = 128; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.thread_ids_num = 1; + test.compute_info.wave_size = wave_size; + test.compute_info.lds_size_dwords = 2; + test.has_compute_info = true; + test.opcodes = {O::V_MOV_B32, O::S_MOV_B32, O::S_MOV_B64, O::S_BARRIER, + O::V_CMP_EQ_U32, O::V_LSHLREV_B32, O::V_ADD_NC_U32, + O::DS_WRITE_B32, O::DS_OR_B64, O::DS_READ_B32, + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + test.required_spirv = {"OpAtomicOr %ulong", "WorkgroupMemoryExplicitLayoutKHR", "Aliased"}; + return test; +} + TestCase DsBoundedAtomicBoundaries(bool decrement, bool gds) { using O = ShaderOpcode; constexpr u32 guard = 0x13579bdfu; @@ -30392,6 +30488,8 @@ std::vector MakeCases() { AddCase(DsWideGdsPartialBounds); AddCase(DsAtomicNoReturnVariants); AddCase(DsAtomicReturnVariants); + AddCase(DsOrB64Bounds); + for (u32 wave_size : {32, 64}) cases.push_back(DsOrB64Contention(wave_size)); for (bool decrement : {false, true}) { for (bool gds : {false, true}) { cases.push_back(DsBoundedAtomicBoundaries(decrement, gds)); @@ -32933,6 +33031,15 @@ void CheckShaderRecompilerFatalContracts() { (void)CompileCase(test); }); + ExpectFatal("GdsOrB64Rejection", [] { + TestCase test; + test.name = "GdsOrB64Rejection"; + test.code = {EncodeDs0(0x4a, 0, true), EncodeDs1(0, 3, 7)}; + AppendEnd(&test.code); + test.opcodes = {ShaderOpcode::DS_OR_B64, ShaderOpcode::S_ENDPGM}; + (void)CompileCase(test); + }); + { ShaderRecompiler::IR::Program program; program.resource_tracking_complete = true;