diff --git a/src/graphics/host_gpu/graphicContext.h b/src/graphics/host_gpu/graphicContext.h index 172890845..d579eeee9 100644 --- a/src/graphics/host_gpu/graphicContext.h +++ b/src/graphics/host_gpu/graphicContext.h @@ -29,6 +29,7 @@ struct GraphicContext { bool memory_budget_ext_enabled = false; bool compute_subgroup_size_control_enabled = false; bool sample_rate_shading_enabled = false; + bool shader_image_int64_atomics_enabled = false; // bool fp64_denorm_preserve = false; // Temporarily disabled. bool attachment_feedback_loop_enabled = false; bool provoking_vertex_last_enabled = false; diff --git a/src/graphics/host_gpu/renderer/image/imageView.cpp b/src/graphics/host_gpu/renderer/image/imageView.cpp index b662523b1..8e67580bf 100644 --- a/src/graphics/host_gpu/renderer/image/imageView.cpp +++ b/src/graphics/host_gpu/renderer/image/imageView.cpp @@ -358,10 +358,8 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) { } vk::ImageViewUsageCreateInfo usage {}; - usage.usage = image.usage; - if (!is_storage) { - usage.usage &= ~vk::ImageUsageFlagBits::eStorage; - } + usage.usage = is_storage ? vk::ImageUsageFlagBits::eStorage + : image.usage & ~vk::ImageUsageFlagBits::eStorage; vk::ImageViewMinLodCreateInfoEXT min_lod {}; if (normalized.min_lod != 0) { min_lod.minLod = static_cast(normalized.base_level) + diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index f4a91b782..f2c8adda5 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -312,13 +312,14 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource, resource.written && !resource.read && !resource.atomic; const auto numeric_class = Prospero::SampledTextureNumericClass(format); const bool raw_float_atomic = format == Prospero::BufferFormat::k32Float && uint_resource && - resource.atomic; + resource.atomic && !resource.atomic64; const bool format_ok = raw_sint_storage || raw_float_atomic || (numeric_class != Prospero::TextureNumericClass::Unsupported && numeric_class != Prospero::TextureNumericClass::Sint && uint_resource == (numeric_class == Prospero::TextureNumericClass::Uint) && - (!resource.atomic || format == Prospero::BufferFormat::k32UInt)); + (!resource.atomic || format == (resource.atomic64 ? Prospero::BufferFormat::k32_32UInt + : Prospero::BufferFormat::k32UInt))); if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) { return; } @@ -354,7 +355,8 @@ static TextureCache::ImageDesc NullTextureDesc(const ShaderRecompiler::IR::Image desc.info.guest_format = Prospero::BufferFormat::k32Float; break; case Prospero::TextureNumericClass::Uint: - desc.info.guest_format = Prospero::BufferFormat::k32UInt; + desc.info.guest_format = resource.atomic64 ? Prospero::BufferFormat::k32_32UInt + : Prospero::BufferFormat::k32UInt; break; case Prospero::TextureNumericClass::Sint: desc.info.guest_format = Prospero::BufferFormat::k32SInt; @@ -365,10 +367,10 @@ static TextureCache::ImageDesc NullTextureDesc(const ShaderRecompiler::IR::Image desc.info.type = Prospero::ImageType::kColor2D; desc.info.extent = {1, 1, 1}; desc.info.resources = {1, 1}; - desc.info.bytes_per_block = 4; + desc.info.bytes_per_block = Prospero::NumBytesPerElement(desc.info.guest_format); desc.info.samples = 1; desc.info.mip_layout[0] = {0, 0, 1, 1}; - desc.view_info.format = desc.info.pixel_format; + desc.view_info.format = resource.atomic64 ? vk::Format::eR64Uint : desc.info.pixel_format; desc.view_info.type = vk::ImageViewType::e2D; desc.view_info.aspect = vk::ImageAspectFlagBits::eColor; desc.view_info.usage = binding == TextureCache::BindingType::Storage @@ -538,6 +540,9 @@ static bool ResolveTextureMipView(const TileSurfaceDescription& description, boo TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, const ShaderRecompiler::IR::DescriptorValue& value) { + if (resource.atomic64 && !m_context.GetGraphics().shader_image_int64_atomics_enabled) { + EXIT("64-bit image atomics require shaderImageInt64Atomics\n"); + } auto descriptor = DecodeNativeDescriptor(value); const bool storage = resource.written; if (storage) { @@ -650,8 +655,9 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR pixel_format = depth_format->depth_attachment_format; } } - const auto storage_view_format = storage && (resource.atomic || - format == Prospero::BufferFormat::k32SInt) + const auto storage_view_format = resource.atomic64 ? vk::Format::eR64Uint + : storage && (resource.atomic || + format == Prospero::BufferFormat::k32SInt) ? vk::Format::eR32Uint : SrgbStorageViewFormat(pixel_format); const auto view_format = storage && storage_view_format != vk::Format::eUndefined diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index a143f66d9..d05a85d5a 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -460,7 +460,15 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, provoking_vertex.pNext = supported_features2.pNext; supported_features2.pNext = &provoking_vertex; } + const bool image_atomic_int64_extension = + HasExtension(device_extensions, VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME); + vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic_int64 {}; + if (image_atomic_int64_extension) { + image_atomic_int64.pNext = supported_features2.pNext; + supported_features2.pNext = &image_atomic_int64; + } physical_device.getFeatures2(&supported_features2); + graphics.shader_image_int64_atomics_enabled = image_atomic_int64.shaderImageInt64Atomics; auto features12 = WindowContext::RequiredVulkan12Features(); features12.shaderSharedInt64Atomics = supported_features12.shaderSharedInt64Atomics; @@ -605,6 +613,11 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, provoking_vertex.transformFeedbackPreservesProvokingVertex = VK_FALSE; create_info.pNext = &provoking_vertex; } + if (graphics.shader_image_int64_atomics_enabled) { + image_atomic_int64.pNext = const_cast(create_info.pNext); + image_atomic_int64.sparseImageInt64Atomics = VK_FALSE; + create_info.pNext = &image_atomic_int64; + } create_info.pQueueCreateInfos = &queue_create_info; create_info.queueCreateInfoCount = 1; create_info.enabledExtensionCount = static_cast(device_extensions.size()); @@ -969,6 +982,7 @@ void WindowContext::CreateVulkan() { graphic_ctx.memory_budget_ext_enabled = true; } for (const auto* extension: {VK_EXT_ROBUSTNESS_2_EXTENSION_NAME, + VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME, VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME, VK_EXT_MESH_SHADER_EXTENSION_NAME, VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 2f12d0ec1..179dda7cd 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -124,13 +124,21 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) { sampled = 2; if (image.atomic) { EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint); - format = spv::ImageFormatR32ui; + format = image.atomic64 ? spv::ImageFormatR64ui : spv::ImageFormatR32ui; + if (image.atomic64) { + state.builder.RequireExtension("SPV_EXT_shader_image_int64"); + state.builder.RequireCapability(spv::CapabilityInt64); + state.builder.RequireCapability(spv::CapabilityInt64Atomics); + state.builder.RequireCapability(spv::CapabilityInt64ImageEXT); + } } } else { EXIT("invalid image resource class"); } const auto& info = ImageDimensionInfoFor(image.dimension); - return state.builder.Type(spv::OpTypeImage, ImageScalarType(state, image.numeric_class), + const auto scalar_type = image.atomic64 ? TypeScalarU64(state) + : ImageScalarType(state, image.numeric_class); + return state.builder.Type(spv::OpTypeImage, scalar_type, info.spirv_dimension, image.depth_compare ? 1u : 0u, info.arrayed, info.multisampled, sampled, format); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 14dfae8f2..eb008c54d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -865,10 +865,14 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { } if (image_info.access == IR::ImageAccess::Atomic) { const auto dimension = image.dimension; - ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { + const auto result_type = TypeId(state, inst.GetType()); + const auto zero = image.atomic64 ? ConstantU64(state, 0) : ConstantU32(state, 0); + ctx.Define(inst, EmitValueOrDefaultIfCondition( + state, ctx.Arg(inst, inst.NumArgs() - 1), result_type, zero, [&]() { const auto pointer = state.builder.AllocateId(); const auto pointer_type = state.builder.Type( - spv::OpTypePointer, spv::StorageClassImage, TypeU32(state)); + spv::OpTypePointer, spv::StorageClassImage, + image.atomic64 ? TypeScalarU64(state) : TypeU32(state)); state.builder.AddFunction(spv::OpImageTexelPointer, pointer_type, pointer, StorageImageDescriptorPointer(state, mem.resource), CoordU32(ctx, mem, *address, dimension), diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h index 645f4998f..c6f3a4d9d 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInstructions.h @@ -317,6 +317,7 @@ inline constexpr auto EmitImageAtomicSMin32 = EmitImage; inline constexpr auto EmitImageAtomicUMin32 = EmitImage; inline constexpr auto EmitImageAtomicSMax32 = EmitImage; inline constexpr auto EmitImageAtomicUMax32 = EmitImage; +inline constexpr auto EmitImageAtomicUMax64 = EmitImage; inline constexpr auto EmitImageAtomicAnd32 = EmitImage; inline constexpr auto EmitImageAtomicOr32 = EmitImage; inline constexpr auto EmitImageAtomicXor32 = EmitImage; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 275b3d96f..323cfaabd 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -1044,17 +1044,11 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { return EmitValueOrDefaultIfCondition( state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0), [&]() { - const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), - ctx.Arg(inst, inst.NumArgs() - 2)); - const auto old = state.builder.AllocateId(); - state.builder.AddFunction( - SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old, - EmitStorageBufferElementPointer(state, resource, index, - TypeStorageBufferU64ElementPointer(state)), - ConstantU32(state, spv::ScopeDevice), - ConstantU32(state, spv::MemorySemanticsMaskNone), value); + const auto pointer = EmitStorageBufferElementPointer( + state, resource, index, TypeStorageBufferU64ElementPointer(state)); + const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice); EmitAtomicMemoryBarrier(state, mem.kind); - return Unary(state, spv::OpBitcast, TypeU64(state), old); + return old; }); }); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 9ec11028c..39f0aaea3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -330,6 +330,7 @@ spv::Op SpirvAtomicOpcode(IR::ValueOpcode opcode) { case IR::ValueOpcode::BufferAtomicSMax32: case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax; case IR::ValueOpcode::ImageAtomicUMax32: + case IR::ValueOpcode::ImageAtomicUMax64: case IR::ValueOpcode::BufferAtomicUMax32: case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax; case IR::ValueOpcode::ImageAtomicAnd32: @@ -352,6 +353,8 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32 uint32_t scope) { const auto opcode = SpirvAtomicOpcode(inst.GetOpcode()); const auto old = ctx.state.builder.AllocateId(); + const bool wide = inst.GetType() == IR::Type::U64; + const auto type = wide ? TypeScalarU64(ctx.state) : TypeU32(ctx.state); if (opcode == spv::OpAtomicCompareExchange) { const auto desired = ctx.Arg(inst, inst.NumArgs() - 3); const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2); @@ -360,12 +363,13 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32 ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator); } else { - const auto value = ctx.Arg(inst, inst.NumArgs() - 2); - ctx.state.builder.AddFunction(opcode, TypeU32(ctx.state), old, pointer, + auto value = ctx.Arg(inst, inst.NumArgs() - 2); + if (wide) value = Unary(ctx.state, spv::OpBitcast, type, value); + ctx.state.builder.AddFunction(opcode, type, old, pointer, ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), value); } - return old; + return wide ? Unary(ctx.state, spv::OpBitcast, TypeU64(ctx.state), old) : old; } void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) { diff --git a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp index 554c0c8ae..1810a4db4 100644 --- a/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp +++ b/src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp @@ -360,9 +360,19 @@ void DecodeMimg(uint32_t pc, std::span code, uint32_t word_index SetUnsupported(inst, Family::MIMG, opcode, "MIMG image gather requires exactly one dmask bit"); } - if (inst.opcode == Opcode::IMAGE_ATOMIC_CMPSWAP && inst.dmask != 0x3u) { - SetUnsupported(inst, Family::MIMG, opcode, - "MIMG image compare-and-swap requires 32-bit DMASK 0x3"); + if (atomic != nullptr) { + const bool compare_swap = inst.opcode == Opcode::IMAGE_ATOMIC_CMPSWAP; + const auto mask32 = compare_swap ? 0x3u : 0x1u; + const auto mask64 = compare_swap ? 0xfu : 0x3u; + if (inst.dmask != mask32 && inst.dmask != mask64) { + SetUnsupported(inst, Family::MIMG, opcode, "MIMG image atomic has invalid DMASK"); + } else if (inst.dmask == mask64) { + inst.data_bits = 64u; + if (inst.opcode != Opcode::IMAGE_ATOMIC_UMAX) { + SetUnsupported(inst, Family::MIMG, opcode, + "MIMG 64-bit image atomic opcode is not implemented"); + } + } } const bool supports_d16 = sample != nullptr || gather != nullptr || opcode == 0x00u || opcode == 0x01u || opcode == 0x08u || opcode == 0x09u; diff --git a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp index c535ffb34..2b51cf59a 100644 --- a/src/graphics/shader/recompiler/frontend/translate/Memory.cpp +++ b/src/graphics/shader/recompiler/frontend/translate/Memory.cpp @@ -568,7 +568,9 @@ void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode ir.GetExec()}, flags); } else { - result = ir.Emit(opcode, {resource, address, ReadU32(data), ir.GetExec()}, flags); + const IR::Value value = + memory.data_bits == 64u ? IR::Value(ReadU64(data)) : IR::Value(ReadU32(data)); + result = ir.Emit(opcode, {resource, address, value, ir.GetExec()}, flags); } if (inst.glc) { WriteOperand(inst.dst, result); @@ -1052,7 +1054,8 @@ void Translator::EmitMemory(const Decoder::Instruction& inst) { case Decoder::Opcode::IMAGE_ATOMIC_SMAX: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicSMax32); case Decoder::Opcode::IMAGE_ATOMIC_UMAX: - return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicUMax32); + return IMAGE_ATOMIC(inst, inst.data_bits == 64u ? IR::ValueOpcode::ImageAtomicUMax64 + : IR::ValueOpcode::ImageAtomicUMax32); case Decoder::Opcode::IMAGE_ATOMIC_AND: return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicAnd32); case Decoder::Opcode::IMAGE_ATOMIC_OR: diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index 505abab32..0a296c5dc 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -133,6 +133,7 @@ struct ImageResource { bool read = false; bool written = false; bool atomic = false; + bool atomic64 = false; bool depth_compare = false; bool cube = false; bool r128 = false; @@ -278,7 +279,7 @@ struct StageOutput { inline constexpr uint32_t FirstImageBinding = 1u; inline constexpr uint32_t FirstComparisonImageBinding = 22u; inline constexpr uint32_t FirstStorageImageBinding = 29u; -inline constexpr uint32_t ImageBindingCount = 43u; +inline constexpr uint32_t ImageBindingCount = 48u; enum class DescriptorBindingKind : uint32_t { Buffers = 0u, @@ -291,8 +292,8 @@ enum class DescriptorBindingKind : uint32_t { Count, }; -static_assert(static_cast(DescriptorBindingKind::Samplers) == 44u); -static_assert(static_cast(DescriptorBindingKind::Count) == 50u); +static_assert(static_cast(DescriptorBindingKind::Samplers) == 49u); +static_assert(static_cast(DescriptorBindingKind::Count) == 55u); struct PushData { static constexpr uint32_t DwordCount = 32; @@ -369,7 +370,7 @@ DescriptorBindingForImage(const ImageResource& image) { if (image.numeric_class != Prospero::TextureNumericClass::Uint) { return std::nullopt; } - base = AtomicUintBinding; + base = AtomicUintBinding + (image.atomic64 ? 5u : 0u); } else { switch (image.numeric_class) { case Prospero::TextureNumericClass::Float: base = StorageFloatBinding; break; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index a6723372a..767767b76 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -227,6 +227,7 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) { case ValueOpcode::ImageAtomicUMin32: case ValueOpcode::ImageAtomicSMax32: case ValueOpcode::ImageAtomicUMax32: + case ValueOpcode::ImageAtomicUMax64: case ValueOpcode::ImageAtomicAnd32: case ValueOpcode::ImageAtomicOr32: case ValueOpcode::ImageAtomicXor32: diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc index 9623c86e5..f0213607f 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc @@ -305,6 +305,7 @@ VALUE_OPCODE(ImageAtomicSMin32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMin32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicSMax32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1) +VALUE_OPCODE(ImageAtomicUMax64, U64, ImageResource, ImageAddress, U64, U1) VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1) VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1) diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 29c547f39..694c14579 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -460,8 +460,10 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna image.cube = DescriptorIsCube(descriptor); const auto format = static_cast((descriptor.dwords[1] >> 20u) & 0x1ffu); - if (base.atomic && format != Prospero::BufferFormat::k32UInt && - format != Prospero::BufferFormat::k32Float) { + if (base.atomic && + (base.atomic64 ? format != Prospero::BufferFormat::k32_32UInt + : format != Prospero::BufferFormat::k32UInt && + format != Prospero::BufferFormat::k32Float)) { return SpecializationFail( fmt::format("atomic image descriptor {} uses unsupported format {}", i, static_cast(format))); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index f001a0035..1d26dd05f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -1725,6 +1725,8 @@ private: uint32_t AddImage(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) { const auto resource_class = ImageOpcodeInfoOf(op).resource_class; + const bool atomic64 = ImageOpcodeInfoOf(op).access == ImageAccess::Atomic && + memory.data_bits == 64u; const auto mip = resource_class == ImageResourceClass::Storage && memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None; @@ -1733,7 +1735,8 @@ private: auto& image = m_info.images[i]; if (image.source == source && image.resource_class == resource_class && image.dimension == memory.image_dimension && image.mip_mode == mip && - image.depth_compare == depth && image.r128 == memory.image_r128) { + image.depth_compare == depth && image.r128 == memory.image_r128 && + image.atomic64 == atomic64) { Merge(image, op, pc); return i; } @@ -1749,6 +1752,7 @@ private: image.mip_mode = mip; image.depth_compare = depth; image.r128 = memory.image_r128; + image.atomic64 = atomic64; Merge(image, op, pc); m_info.images.push_back(image); return static_cast(m_info.images.size() - 1); diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index 19d703087..7a78c271e 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -3139,15 +3139,15 @@ void TestShaderInfoAndBindingLayout() { void TestImageBindingAbi() { using NumericClass = Libs::Graphics::Prospero::TextureNumericClass; - Check(ImageBindingCount == 43u && + Check(ImageBindingCount == 48u && static_cast(DescriptorBindingKind::Buffers) == 0u && - static_cast(DescriptorBindingKind::Samplers) == 44u && - static_cast(DescriptorBindingKind::Gds) == 45u && - static_cast(DescriptorBindingKind::BdaPagetable) == 46u && - static_cast(DescriptorBindingKind::FaultBuffer) == 47u && - static_cast(DescriptorBindingKind::FlattenedSrt) == 48u && - static_cast(DescriptorBindingKind::ShaderData) == 49u && - static_cast(DescriptorBindingKind::Count) == 50u, + static_cast(DescriptorBindingKind::Samplers) == 49u && + static_cast(DescriptorBindingKind::Gds) == 50u && + static_cast(DescriptorBindingKind::BdaPagetable) == 51u && + static_cast(DescriptorBindingKind::FaultBuffer) == 52u && + static_cast(DescriptorBindingKind::FlattenedSrt) == 53u && + static_cast(DescriptorBindingKind::ShaderData) == 54u && + static_cast(DescriptorBindingKind::Count) == 55u, "native descriptor binding anchors changed"); const std::array sampled_dimensions{ @@ -3170,12 +3170,13 @@ void TestImageBindingAbi() { uint32_t index = 0; const auto CheckBinding = [&](ImageResourceClass resource_class, NumericClass numeric_class, - Decoder::ImageDimension dimension, bool atomic, bool comparison = false) { + Decoder::ImageDimension dimension, bool atomic, bool comparison = false, bool atomic64 = false) { ImageResource image; image.resource_class = resource_class; image.numeric_class = numeric_class; image.dimension = dimension; image.atomic = atomic; + image.atomic64 = atomic64; image.depth_compare = comparison; const auto kind = DescriptorBindingForImage(image); Check(kind.has_value() && @@ -3210,6 +3211,10 @@ void TestImageBindingAbi() { CheckBinding(ImageResourceClass::Storage, NumericClass::Uint, dimension, true); } + for (const auto dimension : storage_dimensions) { + CheckBinding(ImageResourceClass::Storage, NumericClass::Uint, dimension, + true, false, true); + } Check(index == ImageBindingCount, "image descriptor ABI case count changed"); const auto Invalid = [](ImageResource image) { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 233eff1c9..0b3b519df 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -13541,14 +13541,16 @@ public: Image CreateImage2D(const char *shader_name, u32 width, u32 height, vk::Format format, vk::ImageUsageFlags usage, const std::vector &initial, u32 dwords_per_pixel, - vk::ImageLayout final_layout) { + vk::ImageLayout final_layout, + vk::Format view_format = vk::Format::eUndefined) { std::vector> mips; if (!initial.empty()) { mips.push_back(initial); } return CreateImageMips(shader_name, width, height, format, usage, mips, dwords_per_pixel, final_layout, vk::ImageType::e2D, - vk::ImageViewType::e2D, 1); + vk::ImageViewType::e2D, 1, 0, 0, + vk::SampleCountFlagBits::e1, view_format); } static u32 MipExtent(u32 value, u32 level) { @@ -13572,7 +13574,8 @@ public: vk::ImageType image_type, vk::ImageViewType view_type, u32 layers, u32 view_base_layer = 0, u32 view_layers = 0, - vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1) { + vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1, + vk::Format view_format = vk::Format::eUndefined) { Image ret; ret.format = format; ret.width = width; @@ -13596,6 +13599,9 @@ public: vk::ImageUsageFlagBits::eTransferSrc; image_info.sharingMode = vk::SharingMode::eExclusive; image_info.initialLayout = vk::ImageLayout::eUndefined; + if (view_format != vk::Format::eUndefined && view_format != format) { + image_info.flags |= vk::ImageCreateFlagBits::eMutableFormat; + } RequireVk(shader_name, "dispatch", m_device.createImage(&image_info, nullptr, &ret.image), "vkCreateImage"); @@ -13626,7 +13632,7 @@ public: view_info.sType = vk::StructureType::eImageViewCreateInfo; view_info.image = ret.image; view_info.viewType = view_type; - view_info.format = format; + view_info.format = view_format == vk::Format::eUndefined ? format : view_format; view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; view_info.subresourceRange.baseMipLevel = 0; view_info.subresourceRange.levelCount = ret.mip_levels; @@ -16421,6 +16427,7 @@ private: m_runtime_context.physical_device_memory_properties = m_memory_properties; m_runtime_context.queue_family = m_queue_family; m_runtime_context.queue = m_queue; + m_runtime_context.shader_image_int64_atomics_enabled = true; m_runtime_context.attachment_feedback_loop_enabled = m_rasterization_supported; m_runtime_context.provoking_vertex_last_enabled = m_rasterization_supported; const vk::PhysicalDeviceImageFormatInfo2 block_texel_view_info{ @@ -16502,6 +16509,8 @@ private: features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{}; features12.pNext = &workgroup_layout; + vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{}; + workgroup_layout.pNext = &image_atomic64; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR; barycentric.pNext = &features12; @@ -16515,6 +16524,7 @@ private: features12.shaderOutputViewportIndex != true || features12.shaderBufferInt64Atomics != true || features12.shaderSharedInt64Atomics != true || + image_atomic64.shaderImageInt64Atomics != true || workgroup_layout.workgroupMemoryExplicitLayout != true || features12.bufferDeviceAddress != true) { continue; @@ -16529,7 +16539,7 @@ private: } } Require("VulkanHarness", "dispatch", m_physical_device != nullptr, - "no Vulkan graphics+compute device with fragment barycentrics and 64-bit LDS atomics"); + "no Vulkan graphics+compute device with fragment barycentrics and 64-bit LDS/image atomics"); m_physical_device.getMemoryProperties(&m_memory_properties); vk::PhysicalDeviceFeatures available_features{}; @@ -16619,6 +16629,9 @@ private: vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{}; workgroup_layout.workgroupMemoryExplicitLayout = true; device_features12.pNext = &workgroup_layout; + vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{}; + image_atomic64.shaderImageInt64Atomics = true; + workgroup_layout.pNext = &image_atomic64; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; barycentric.sType = vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR; barycentric.pNext = &device_features12; @@ -16674,6 +16687,7 @@ private: VK_KHR_COMPUTE_SHADER_DERIVATIVES_EXTENSION_NAME, VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME, VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME, + VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME, VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME}; if (m_rasterization_supported) { device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME); @@ -17124,15 +17138,20 @@ void RunCase(VulkanHarness *vulkan, const TestCase &test) { test.sampled_image_view_layers); } if (needs_storage_image) { + const bool atomic64 = std::ranges::any_of( + compiled.program.info.images, + [](const auto &image) { return image.atomic64; }); storage_image = vulkan->CreateImage2D( test.name, test.image_width, test.image_height, test.storage_image_format, vk::ImageUsageFlagBits::eStorage, test.storage_image_rgba, test.storage_image_dwords_per_pixel, vk::ImageLayout::eGeneral); storage_image_uint = vulkan->CreateImage2D( - test.name, test.image_width, test.image_height, vk::Format::eR32Uint, - vk::ImageUsageFlagBits::eStorage, test.storage_image_r32ui, 1, - vk::ImageLayout::eGeneral); + test.name, test.image_width, test.image_height, + atomic64 ? vk::Format::eR32G32Uint : vk::Format::eR32Uint, + vk::ImageUsageFlagBits::eStorage, test.storage_image_r32ui, atomic64 ? 2 : 1, + vk::ImageLayout::eGeneral, + atomic64 ? vk::Format::eR64Uint : vk::Format::eR32Uint); } if (needs_sampler) { sampler = vulkan->CreateNearestSampler(test.name, sampled_image.mip_levels); @@ -30701,6 +30720,76 @@ TestCase ImageAtomicVariants() { return test; } +template TestCase ImageAtomicUMaxGlcAndExec() { + // Input and old texel: high DWORD ordering, unsigned sign boundary, and + // equal-high-DWORD ordering must all survive the RG32_UINT -> R64_UINT view. + constexpr uint64_t values[][2] = { + {0x0000000100000001ull, 0x00000000ffffffffull}, + {0x00000000ffffffffull, 0x0000000100000001ull}, + {0x8000000000000000ull, 0x7fffffffffffffffull}, + {0x12345678ffffffffull, 0x1234567800000001ull}, + {0xffffffffffffffffull, 0x8000000000000000ull}, + }; + constexpr u32 texel_dwords = wide ? 2 : 1; + TestCase test; + test.name = wide ? "ImageAtomicUMax64CapturedGlcAndExec" + : "ImageAtomicUMax32GlcAndExec"; + test.user_data = MakeStorageTextureData(wide ? Prospero::BufferFormat::k32_32UInt + : Prospero::BufferFormat::k32UInt); + test.has_user_data = true; + test.storage_image_r32ui.assign(16 * texel_dwords, 0); + test.expected_storage_image_r32ui = test.storage_image_r32ui; + auto &code = test.code; + code.push_back(EncodeSop1(0x04, 12, 126)); // Preserve EXEC. + for (u32 i = 0; i < std::size(values); ++i) { + const auto input = values[i][0]; + const auto old = values[i][1]; + const bool glc = i != 0 && i != 4; + AppendVMovU32(&code, 0, i & 3u); + AppendVMovU32(&code, 1, i >> 2u); + AppendVMovLiteral(&code, 2, static_cast(input)); + AppendVMovLiteral(&code, 3, static_cast(input >> 32u)); + // Captured PS5 instruction: f05c0308 00000200, DMASK=3, VDATA=v[2:3]. + code.insert(code.end(), {0xf05c0108u | (wide ? 0x200u : 0u) | + (glc ? 0x2000u : 0u), + 0x00000200u}); + AppendStoreVgpr(&code, 2, i * 2); + AppendStoreVgpr(&code, 3, i * 2 + 1); + test.expected.push_back(static_cast(glc ? old : input)); + test.expected.push_back(static_cast((wide && glc ? old : input) >> 32u)); + const uint64_t result = wide ? std::max(input, old) + : std::max(static_cast(input), + static_cast(old)); + for (u32 word = 0; word < texel_dwords; ++word) { + test.storage_image_r32ui[i * texel_dwords + word] = + static_cast(old >> (word * 32u)); + test.expected_storage_image_r32ui[i * texel_dwords + word] = + static_cast(result >> (word * 32u)); + } + } + AppendVMovU32(&code, 0, 1); + AppendVMovU32(&code, 1, 1); // Untouched zero texel, so an EXEC leak is observable. + AppendVMovLiteral(&code, 2, 0xabcdef01u); + AppendVMovLiteral(&code, 3, 0x87654321u); + code.push_back(EncodeSop1(0x04, 126, InlineU32(0))); + code.insert(code.end(), {0xf05c2108u | (wide ? 0x200u : 0u), 0x00000200u}); + code.push_back(EncodeSop1(0x04, 126, 12)); + AppendStoreVgpr(&code, 2, std::size(values) * 2); + AppendStoreVgpr(&code, 3, std::size(values) * 2 + 1); + test.expected.insert(test.expected.end(), {0xabcdef01u, 0x87654321u}); + AppendEnd(&code); + test.opcodes = {ShaderOpcode::V_MOV_B32, ShaderOpcode::S_MOV_B64, + ShaderOpcode::IMAGE_ATOMIC_UMAX, ShaderOpcode::BUFFER_STORE_DWORD, + ShaderOpcode::S_ENDPGM}; + test.required_spirv = {"OpAtomicUMax", "OpImageTexelPointer", + wide ? "R64ui" : "R32ui"}; + if constexpr (wide) { + test.required_spirv.push_back("Int64ImageEXT"); + test.required_spirv.push_back("SPV_EXT_shader_image_int64"); + } + return test; +} + template TestCase ImageAtomicSignedMinMax() { using O = ShaderOpcode; // Input, old value, signed minimum, signed maximum. @@ -31734,6 +31823,8 @@ std::vector MakeCases() { AddCase(ImageAtomicSwapReturnsPreviousTexel); AddCase(ImageStoreAndAtomicUseSeparateBindings); AddCase(ImageAtomicVariants); + AddCase(ImageAtomicUMaxGlcAndExec); + AddCase(ImageAtomicUMaxGlcAndExec); AddCase(ImageAtomicFloatGlcAndExec); AddCase(ImageAtomicFloatGlcAndExec); AddCase(ImageAtomicFloatSpecialValues); @@ -36769,6 +36860,12 @@ int main(int argc, char **argv) { RunCase(&vulkan, ImageAtomicVariants()); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--image-atomic64-only") == 0) { + VulkanHarness vulkan; + RunCase(&vulkan, ImageAtomicUMaxGlcAndExec()); + RunCase(&vulkan, ImageAtomicUMaxGlcAndExec()); + return 0; + } if (argc == 2 && std::strcmp(argv[1], "--ds-atomics-only") == 0) { VulkanHarness vulkan; for (const auto &test : MakeCases()) { diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index fc7dcdfa6..82a75b47d 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -4269,6 +4269,38 @@ void TestScalarAshrI64Decoder() { "S_ASHR_I64 is missing from the decoded dump"); } +void TestImageAtomicWidthDecoder() { + using namespace ShaderRecompiler::Decoder; + // Captured PPSA29343 pixel shader: image_atomic_umax v[2:3], v[0:1], s[0:7]. + const uint32_t captured[] = {0xf05c0308u, 0x00000200u}; + Instruction decoded; + DecodeInstruction(captured, 0, decoded); + Check(decoded.opcode == Opcode::IMAGE_ATOMIC_UMAX && + decoded.data_bits == 64u && decoded.data_dwords == 2u && + decoded.dst.reg == 2u && decoded.dmask == 3u && !decoded.glc, + "captured image atomic lost its 64-bit data width"); + + for (const auto opcode : {0x10u, 0x17u}) { + for (uint32_t mask = 0; mask < 16u; ++mask) { + const uint32_t words[] = {EncodeMimg0(opcode, mask), captured[1]}; + DecodeInstruction(words, 0, decoded); + const bool compare_swap = opcode == 0x10u; + const bool supported = mask == (compare_swap ? 3u : 1u) || + (!compare_swap && mask == 3u); + Check((decoded.opcode != Opcode::UNSUPPORTED) == supported, + "image atomic accepted an invalid or unsupported width mask"); + if (supported) { + Check(decoded.data_bits == (!compare_swap && mask == 3u ? 64u : 32u), + "image atomic DMASK selected the wrong data width"); + } + } + } + const uint32_t unsupported[] = {EncodeMimg0(0x11u, 3u), captured[1]}; + DecodeInstruction(unsupported, 0, decoded); + Check(decoded.opcode == Opcode::UNSUPPORTED && decoded.data_bits == 64u, + "unsupported 64-bit image atomic silently decoded as 32-bit"); +} + void TestNewShaderDecoderArchitecture() { using namespace ShaderRecompiler::Decoder; @@ -14301,6 +14333,7 @@ int main() { // ShaderRecompilerComputeTests; keep the distinct decoder contract checks // here. TestScalarAshrI64Decoder(); + TestImageAtomicWidthDecoder(); TestNewShaderDecoderArchitecture(); TestImageAddressOperands(); TestSopkCompareImmediateExtension();