From c3f4962715bc2cdaad33665a4edca8eae6a567a8 Mon Sep 17 00:00:00 2001 From: nmzik Date: Sun, 4 Oct 2026 14:02:15 +0400 Subject: [PATCH] shader: use native narrow storage for byte-aligned PS5 buffers Co-authored-by: MehmetCambaz --- .../renderer/pipeline/descriptors.cpp | 2 +- .../presentation/window/vulkanWindow.cpp | 17 ++ .../presentation/window/windowInternal.h | 1 + .../recompiler/backend/spirv/SpirvEmitter.cpp | 3 + .../backend/spirv/spirvEmitterInternal.h | 18 +- .../backend/spirv/spirvEmitterMemory.cpp | 133 +++++++------ .../spirv/spirvEmitterMemoryHelpers.cpp | 31 +-- .../backend/spirv/spirvEmitterModule.cpp | 85 ++++---- tests/ShaderRecompilerComputeTests.cpp | 186 +++++++++++++++--- 9 files changed, 321 insertions(+), 155 deletions(-) diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index e592cc6c3..44c645b7f 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -130,7 +130,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource const auto aligned_offset = Common::AlignDown(offset, alignment); const auto adjustment = offset - aligned_offset; const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange; - if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) { + if (adjustment >= 256 || size > max_range - adjustment) { EXIT("storage buffer offset adjustment is unsupported\n"); } buffer_offset = static_cast(adjustment); diff --git a/src/graphics/presentation/window/vulkanWindow.cpp b/src/graphics/presentation/window/vulkanWindow.cpp index 0ddf31621..ab03572e6 100644 --- a/src/graphics/presentation/window/vulkanWindow.cpp +++ b/src/graphics/presentation/window/vulkanWindow.cpp @@ -47,6 +47,12 @@ struct VulkanExtensions { std::vector available_layers; }; +vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept { + vk::PhysicalDeviceVulkan11Features features {}; + features.storageBuffer16BitAccess = VK_TRUE; + return features; +} + vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept { vk::PhysicalDeviceVulkan12Features features {}; features.samplerMirrorClampToEdge = VK_TRUE; @@ -55,6 +61,7 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe features.shaderOutputViewportIndex = VK_TRUE; features.bufferDeviceAddress = VK_TRUE; features.shaderBufferInt64Atomics = VK_TRUE; + features.storageBuffer8BitAccess = VK_TRUE; features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE; return features; } @@ -207,6 +214,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa depth_clip_control.pNext = &depth_clip_enable; vk::PhysicalDeviceVulkan12Features features12 {}; + vk::PhysicalDeviceVulkan11Features features11 {}; #if defined(__APPLE__) features12.pNext = &depth_clip_control; #else @@ -214,6 +222,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa fragment_barycentric.pNext = &depth_clip_control; features12.pNext = &fragment_barycentric; #endif + features11.pNext = features12.pNext; + features12.pNext = &features11; features13.pNext = &features12; device_features2.pNext = &features13; @@ -257,6 +267,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge", required_features12.samplerMirrorClampToEdge); + check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess", + WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess); + check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess", + required_features12.storageBuffer8BitAccess); check_feature(features12.timelineSemaphore, "timelineSemaphore", required_features12.timelineSemaphore); check_feature(features12.shaderOutputLayer, "shaderOutputLayer", @@ -482,6 +496,9 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics, workgroup_layout.pNext = &depth_clip_control; features12.pNext = workgroup_layout_extension ? static_cast(&workgroup_layout) : static_cast(&depth_clip_control); + auto features11 = WindowContext::RequiredVulkan11Features(); + features11.pNext = features12.pNext; + features12.pNext = &features11; if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) { Log::WriteToConsoleAndLog(fmt::format( "WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, " diff --git a/src/graphics/presentation/window/windowInternal.h b/src/graphics/presentation/window/windowInternal.h index 35c874ce0..f2ea83561 100644 --- a/src/graphics/presentation/window/windowInternal.h +++ b/src/graphics/presentation/window/windowInternal.h @@ -34,6 +34,7 @@ struct WindowContext { ~WindowContext(); KYTY_CLASS_NO_COPY(WindowContext); + [[nodiscard]] static vk::PhysicalDeviceVulkan11Features RequiredVulkan11Features() noexcept; [[nodiscard]] static vk::PhysicalDeviceVulkan12Features RequiredVulkan12Features() noexcept; [[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept; [[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept; diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 08b23c78b..7bb8f4c18 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -231,6 +231,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program if (memory.resource >= program.info.buffers.size()) { Fail(program, "buffer operation has invalid resource metadata"); } + const auto bits = StorageBufferElementBits(program, memory); + requirements.buffer_u8 |= bits == 8u; + requirements.buffer_u16 |= bits == 16u; if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { if (program.stage != ShaderType::Compute) { Fail(program, "buffer ADD_TID is only valid for compute shaders"); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 37a758755..424a0cb44 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -70,6 +70,8 @@ struct SpirvRequirements { bool function_scratch = false; bool pixel_valid_mask = false; bool buffer_int64_atomics = false; + bool buffer_u8 = false; + bool buffer_u16 = false; bool shared_int64_atomics = false; bool coherent_buffers = false; bool float64 = false; @@ -108,6 +110,8 @@ struct EmitterState { uint32_t lane_count = 1; uint32_t lane_half = 0; uint32_t storage_buffer_variable = 0; + uint32_t storage_buffer_u8_variable = 0; + uint32_t storage_buffer_u16_variable = 0; uint32_t storage_buffer_u64_variable = 0; std::array memory_byte_offsets {}; uint32_t bda_pagetable_variable = 0; @@ -168,10 +172,9 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components); uint32_t TypeF32Vector(EmitterState& state, uint32_t components); uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee); uint32_t TypeFunction(EmitterState& state); -uint32_t TypeStorageBufferPointer(EmitterState& state); -uint32_t TypeStorageBufferElementPointer(EmitterState& state); -uint32_t TypeStorageBufferU64Pointer(EmitterState& state); -uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state); +uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits); +uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits = 32); +uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32); uint32_t TypePhysicalU32Pointer(EmitterState& state); uint32_t TypePushConstantElementPointer(EmitterState& state); uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords); @@ -395,6 +398,7 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index); uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem); Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem); +uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem); void EmitMemoryOffsets(EmitterState& state); @@ -405,9 +409,8 @@ struct MemoryResourceAccess { IR::ResourceKind kind = IR::ResourceKind::None; uint32_t object_pointer = 0; uint32_t length = 0; - uint32_t index_offset = 0; uint32_t byte_offset = 0; - bool add_index_offset = false; + uint32_t element_bits = 32; spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone; }; @@ -419,9 +422,6 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& st const IR::MemoryInfo& mem, uint32_t variable, uint32_t pointer_type); -uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access, - uint32_t raw_index); - uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, uint32_t index); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 66f58e480..0a596afe3 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -4,6 +4,7 @@ #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include +#include #include namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { @@ -234,7 +235,15 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3 uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { if (mem.kind == IR::ResourceKind::Buffer) { - return BufferByteAddress(ctx, inst, mem); + const auto resource = ResourceForDescriptor(ctx.state, IR::DescriptorBindingKind::Buffers, + mem.resource); + const auto prefix = ctx.state.memory_byte_offsets[resource]; + const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), + BufferByteAddress(ctx, inst, mem), prefix); + // The host binding prefix must not wrap an out-of-range guest address into the buffer. + return Select(ctx.state, TypeU32(ctx.state), + Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), address, prefix), + address, ConstantU32(ctx.state, UINT32_MAX)); } if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { if (mem.offset == 0u) { @@ -254,18 +263,6 @@ uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2)); } -struct PreparedMemoryElement { - MemoryResourceAccess resource; - uint32_t index = 0; -}; - -PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem, - uint32_t raw_index) { - auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index); - return {.resource = resource, .index = index}; -} - uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index); @@ -274,7 +271,7 @@ uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource) { - const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem)); + const auto index = DwordIndex(ctx, inst, mem); return EmitValueOrZeroIfCondition( ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { return LoadWordInBounds(ctx, resource, index); }); @@ -290,10 +287,10 @@ uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t bits, bool sign_extend) { - const auto address = ByteAddress(ctx, inst, mem); - const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 2)); - const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index); + const auto address = ByteAddress(ctx, inst, mem); + const auto index = Binary( + ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address, + ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u))); return EmitValueOrZeroIfCondition( ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { return LoadSubwordInBounds(ctx, resource, address, index, bits, sign_extend); @@ -311,15 +308,23 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) { - const auto word = LoadWordInBounds(ctx, resource, index); - const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 3)); - const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte, - ConstantU32(ctx.state, 3)); - const auto value = - Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), - Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); + uint32_t value; + if (resource.element_bits < 32u) { + const auto loaded = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction(spv::OpLoad, TypeStorageBufferElement(ctx.state, bits), loaded, + EmitMemoryElementPointer(ctx.state, resource, index), + resource.memory_access); + value = Unary(ctx.state, spv::OpUConvert, TypeU32(ctx.state), loaded); + } else { + const auto word = LoadWordInBounds(ctx, resource, index); + const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 3)); + const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte, + ConstantU32(ctx.state, 3)); + value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), + Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); + } if (!sign_extend) return value; const auto left = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), value, ConstantU32(ctx.state, 32u - bits)); @@ -408,6 +413,13 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t address, uint32_t index, uint32_t bits, uint32_t data) { const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index); + if (resource.element_bits < 32u) { + ctx.state.builder.AddFunction( + spv::OpStore, pointer, + Unary(ctx.state, spv::OpUConvert, TypeStorageBufferElement(ctx.state, bits), data), + resource.memory_access); + return; + } const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3)), @@ -435,10 +447,10 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) { - const auto address = ByteAddress(ctx, inst, mem); - const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 2)); - const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index); + const auto address = ByteAddress(ctx, inst, mem); + const auto index = Binary( + ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address, + ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u))); EmitIfCondition( ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data); @@ -505,7 +517,7 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) { void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, const MemoryResourceAccess& resource, uint32_t data) { - const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem)); + const auto index = DwordIndex(ctx, inst, mem); EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { StoreWordInBounds(ctx, resource, index, data); }); @@ -564,10 +576,11 @@ template uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, Fn&& operation) { return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + const auto index = DwordIndex(ctx, inst, mem); + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); return EmitValueOrZeroIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { - return operation(EmitMemoryElementPointer(ctx.state, access.resource, access.index)); + ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { + return operation(EmitMemoryElementPointer(ctx.state, resource, index)); }); }); } @@ -613,25 +626,15 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR:: PreparedFormattedMemory plan; plan.info = info; plan.resource = resource; - auto start = ByteAddress(ctx, inst, mem); - plan.in_bounds = ConstantBool(ctx.state, true); - if (resource.add_index_offset) { - const auto prefixed = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start, - resource.byte_offset); - plan.in_bounds = Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), - prefixed, start); - start = prefixed; - } + const auto start = ByteAddress(ctx, inst, mem); // Check the whole, unaligned format span before aligning the actual memory address. const auto end = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start, ConstantU32(ctx.state, info.byte_size - 1u)); const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end, - ConstantU32(ctx.state, 2)); + ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u))); plan.in_bounds = AndCondition( - ctx.state, plan.in_bounds, - AndCondition(ctx.state, - Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start), - EmitMemoryElementInBounds(ctx.state, plan.resource, last_index))); + ctx.state, Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start), + EmitMemoryElementInBounds(ctx.state, plan.resource, last_index)); plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start, ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u))); return plan; @@ -645,7 +648,7 @@ std::pair FormattedComponentAddress(EmitterState& state, const auto address = offset == 0u ? plan.base : Binary( state, spv::OpIAdd, TypeU32(state), plan.base, ConstantU32(state, offset)); return {address, Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, - ConstantU32(state, 2))}; + ConstantU32(state, std::countr_zero(plan.resource.element_bits / 8u)))}; } uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, @@ -1031,9 +1034,8 @@ uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co ? base : Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, component * 4u)); - const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - const auto index = EmitMemoryElementIndex(state, resource, raw_index); values[component] = EmitValueOrZeroIfCondition( state, EmitMemoryElementInBounds(state, resource, index), [&]() { return LoadWordInBounds(ctx, resource, index); }); @@ -1052,9 +1054,8 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo const auto address = component == 0u ? base : Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, component * 4u)); - const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - const auto index = EmitMemoryElementIndex(state, resource, raw_index); EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&]() { StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, component + 1u)); @@ -1112,7 +1113,7 @@ void DefineGetBdaPointer(EmitterState& state) { ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS)); const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64); const auto entry_pointer = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferU64ElementPointer(state), + state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64), entry_pointer, state.bda_pagetable_variable, ConstantU32(state, 0), page); const auto base = state.builder.AllocateId(); @@ -1160,16 +1161,15 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { return EmitValueOrDefaultIfCondition( state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() { const auto resource = PrepareStorageBufferResourceAccess( - state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state)); - const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state), - ByteAddress(ctx, inst, mem), resource.byte_offset); + state, mem, state.storage_buffer_u64_variable, TypeStorageBufferPointer(state, 64)); + const auto byte_address = ByteAddress(ctx, inst, mem); const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address, ConstantU32(state, 3u)); return EmitValueOrDefaultIfCondition( state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0), [&]() { const auto pointer = EmitStorageBufferElementPointer( - state, resource, index, TypeStorageBufferU64ElementPointer(state)); + state, resource, index, TypeStorageBufferElementPointer(state, 64)); const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice); EmitAtomicMemoryBarrier(state, mem.kind); return old; @@ -1216,16 +1216,17 @@ void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) { const auto& mem = ctx.Memory(inst); const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32; EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + const auto index = DwordIndex(ctx, inst, mem); + const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); EmitIfCondition( - ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() { ctx.state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)); const auto data = ctx.state.builder.AllocateId(); ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable); AtomicUpdate( - ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index), + ctx.state, EmitMemoryElementPointer(ctx.state, resource, index), mem.kind, [&](uint32_t old) { const auto old_f = Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old); @@ -1269,10 +1270,9 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) { Binary(state, spv::OpIAdd, TypeU32(state), base, size), ConstantU32(state, 0xc000u))); } - const auto raw_index = + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); const auto access = PrepareMemoryResourceAccess(state, mem); - const auto index = EmitMemoryElementIndex(state, access, raw_index); const auto exec = ctx.Arg(inst, 1); const auto ballot = ctx.Ballot(inst.Arg(1)); const auto low = state.builder.AllocateId(); @@ -1343,7 +1343,10 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) { index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u)); } const auto access = PrepareMemoryResourceAccess(state, mem); - const auto element = EmitMemoryElementIndex(state, access, index); + // Scalar buffer addressing aligns the base and each offset independently. + const auto element = Binary( + state, spv::OpIAdd, TypeU32(state), index, + Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byte_offset, ConstantU32(state, 2))); const auto condition = EmitMemoryElementInBounds(state, access, element); ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { return EmitNative( diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 0406ad732..de07dd0a0 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -92,6 +92,15 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR:: return state.program.info.buffers[mem.resource].descriptor_format; } +uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) { + if (!mem.formatted) return mem.data_bits; + const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format) + : program.info.buffers[mem.resource].descriptor_format; + const auto info = Format::GetFormatInfo(format); + return info.type == Format::ComponentType::Unknown || info.packed_bitfield + ? 32u : info.component_bits[0]; +} + void EmitMemoryOffsets(EmitterState& state) { for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) { const auto word = @@ -194,24 +203,18 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: EXIT("physical address memory must use the BDA emitter\n"); case IR::ResourceKind::ScalarBuffer: case IR::ResourceKind::Buffer: { + const auto bits = mem.kind == IR::ResourceKind::Buffer + ? StorageBufferElementBits(state.program, mem) : 32u; + const auto variable = bits == 8u ? state.storage_buffer_u8_variable + : bits == 16u ? state.storage_buffer_u16_variable + : state.storage_buffer_variable; access = PrepareStorageBufferResourceAccess( - state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state)); - access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset, - ConstantU32(state, 2u)); - access.add_index_offset = true; + state, mem, variable, TypeStorageBufferPointer(state, bits)); + access.element_bits = bits; return access; } default: EXIT("unsupported memory resource kind: %u\n", static_cast(mem.kind)); } - access.length = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, - access.object_pointer, 0); - return access; -} - -uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access, - uint32_t raw_index) { - return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index; } uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, @@ -240,7 +243,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces return pointer; } return EmitStorageBufferElementPointer(state, access, index, - TypeStorageBufferElementPointer(state)); + TypeStorageBufferElementPointer(state, access.element_bits)); } uint32_t EmitStorageBufferElementPointer(EmitterState& state, diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index 1a6d97d59..287912f84 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -123,46 +123,31 @@ uint32_t TypeFunction(EmitterState& state) { return state.function_type; } -uint32_t StorageRuntimeArrayType(EmitterState& state) { - return state.builder.DecoratedType( - spv::OpTypeRuntimeArray, - {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}}, TypeU32(state)); +uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) { + return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state) + : state.builder.Type(spv::OpTypeInt, bits, 0); } -uint32_t StorageBufferType(EmitterState& state) { +uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) { + return state.builder.DecoratedType( + spv::OpTypeRuntimeArray, + {{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}}, + TypeStorageBufferElement(state, bits)); +} + +uint32_t StorageBufferType(EmitterState& state, uint32_t bits = 32) { return state.builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, {spv::OpDecorate, {spv::DecorationBlock}}}, - StorageRuntimeArrayType(state)); + StorageRuntimeArrayType(state, bits)); } -uint32_t TypeStorageBufferPointer(EmitterState& state) { - return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state)); +uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits) { + return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state, bits)); } -uint32_t TypeStorageBufferElementPointer(EmitterState& state) { - return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state)); -} - -uint32_t StorageU64RuntimeArrayType(EmitterState& state) { - return state.builder.DecoratedType( - spv::OpTypeRuntimeArray, - {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}}, TypeScalarU64(state)); -} - -uint32_t StorageBufferU64Type(EmitterState& state) { - return state.builder.DecoratedType(spv::OpTypeStruct, - {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, - {spv::OpDecorate, {spv::DecorationBlock}}}, - StorageU64RuntimeArrayType(state)); -} - -uint32_t TypeStorageBufferU64Pointer(EmitterState& state) { - return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state)); -} - -uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) { - return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state)); +uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) { + return TypePointer(state, spv::StorageClassStorageBuffer, TypeStorageBufferElement(state, bits)); } uint32_t TypePhysicalU32Pointer(EmitterState& state) { @@ -244,27 +229,34 @@ void DefineDescriptors(EmitterState& state) { case IR::DescriptorBindingKind::Buffers: state.storage_buffer_variable = Define(ArrayType(StorageBufferType(state)), "buffers"); + if (state.requirements.buffer_u8) { + state.storage_buffer_u8_variable = + Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8"); + } + if (state.requirements.buffer_u16) { + state.storage_buffer_u16_variable = + Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16"); + } if (state.requirements.buffer_int64_atomics) { state.storage_buffer_u64_variable = - Define(ArrayType(StorageBufferU64Type(state)), "buffers_u64"); - state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable, - spv::DecorationAliased); - state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable, - spv::DecorationAliased); + Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64"); } - if (state.requirements.coherent_buffers) { + for (const auto variable: {state.storage_buffer_variable, state.storage_buffer_u8_variable, + state.storage_buffer_u16_variable, state.storage_buffer_u64_variable}) { + if (variable == 0) continue; + if (state.storage_buffer_u8_variable != 0 || state.storage_buffer_u16_variable != 0 || + state.storage_buffer_u64_variable != 0) { + state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); + } // RDNA2 stores publish to L2 even without GLC; every alias of the buffer // must participate in visibility for cache-bypassing polling loads. - state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable, - spv::DecorationCoherent); - if (state.storage_buffer_u64_variable != 0) { - state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable, - spv::DecorationCoherent); + if (state.requirements.coherent_buffers) { + state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent); } } break; case IR::DescriptorBindingKind::BdaPagetable: - state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable"); + state.bda_pagetable_variable = Define(StorageBufferType(state, 64), "bda_pagetable"); break; case IR::DescriptorBindingKind::FaultBuffer: state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer"); @@ -669,6 +661,13 @@ void DefineModule(EmitterState& state) { state.builder.RequireCapability(spv::CapabilityShader); state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve); + if (state.requirements.buffer_u8) { + state.builder.RequireExtension("SPV_KHR_8bit_storage"); + state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess); + } + if (state.requirements.buffer_u16) { + state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess); + } if (state.program.info.uses_dma) { state.builder.RequireCapability(spv::CapabilityInt64); state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 406b751a6..49f8f6ab6 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1311,7 +1311,7 @@ struct TestCase { bool has_user_data = false; u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7); bool compile_only = false; - size_t storage_buffer_range_dwords = 0; + size_t storage_buffer_range_bytes = 0; std::vector storage_buffer_offsets; std::vector bda_mappings; bool expand_shader_data_storage = false; @@ -1700,7 +1700,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) { if (resource < test.storage_buffer_offsets.size()) { offset = test.storage_buffer_offsets[resource]; } - Require(test.name, "shader data", offset % sizeof(u32) == 0 && offset < 256, + Require(test.name, "shader data", offset < 256, "storage buffer offset is not representable"); packed_user_data[result.program.bindings.memory_offset_dword + i / 4u] |= offset << ((i % 4u) * 8u); @@ -11360,6 +11360,32 @@ public: "descriptor rebind lost its clamped guest range or retained a stale host owner"); } + { + const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true}); + constexpr uint64_t buffer_address = base + 0xe0002; + ShaderBufferResource descriptor{}; + descriptor.UpdateAddress48(buffer_address); + descriptor.fields[1] |= 6u << 16u; + descriptor.fields[2] = 2; + descriptor.fields[3] = 0x5204; + ShaderRecompiler::IR::ResourceSnapshot snapshot; + auto &value = snapshot.buffers.emplace_back(); + std::copy_n(descriptor.fields, 4, value.dwords.begin()); + value.dword_count = 4; + ShaderStageRuntime runtime{&buffer_program, &snapshot}; + PreparedBindings bindings; + executor.PrepareBindings(runtime, bindings); + executor.FindBuffers(bindings); + executor.RebindBuffers(bindings); + const auto &owner = resources.GetBufferCache().GetBuffer(bindings.buffer_sources[0].id); + Require(name, "halfword buffer base and exact tail", + bindings.buffers[0].buffer == owner.Handle() && + bindings.buffers[0].offset + 2 == owner.Offset(buffer_address) && + bindings.buffers[0].range == 14 && + bindings.shader_data[buffer_program.bindings.memory_offset_dword] == 2, + "native binding lost the byte adjustment or rounded the final halfword range"); + } + constexpr auto stencil_format = Prospero::BufferFormat::k8UInt; constexpr auto linear = Prospero::TileMode::kLinear; TileSizeAlign stencil_layout{}; @@ -14545,13 +14571,13 @@ public: info.buffer = buffer.buffer; info.offset = 0; info.range = buffer.size; - if (test.storage_buffer_range_dwords != 0) { + if (test.storage_buffer_range_bytes != 0) { const auto resource = buffers->resources[i]; const auto offset = resource < test.storage_buffer_offsets.size() ? test.storage_buffer_offsets[resource] : 0u; info.range = static_cast( - test.storage_buffer_range_dwords * sizeof(u32) + offset); + test.storage_buffer_range_bytes + offset); Require(test.name, "dispatch", info.range <= buffer.size, "storage buffer descriptor range exceeds backing buffer"); } @@ -17117,10 +17143,12 @@ private: if ((queues[i].queueFlags & (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) == (vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) { + vk::PhysicalDeviceVulkan11Features features11{}; vk::PhysicalDeviceVulkan12Features features12{}; features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{}; - features12.pNext = &workgroup_layout; + features12.pNext = &features11; + features11.pNext = &workgroup_layout; vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{}; workgroup_layout.pNext = &image_atomic64; vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{}; @@ -17132,6 +17160,8 @@ private: physical.getFeatures2(&features); if (barycentric.fragmentShaderBarycentric != true || features.features.shaderInt64 != true || + features11.storageBuffer16BitAccess != true || + features12.storageBuffer8BitAccess != true || features12.samplerMirrorClampToEdge != true || features12.shaderOutputViewportIndex != true || features12.shaderBufferInt64Atomics != true || @@ -17157,7 +17187,9 @@ private: vk::PhysicalDeviceFeatures available_features{}; m_physical_device.getFeatures(&available_features); + vk::PhysicalDeviceVulkan11Features available_features11{}; vk::PhysicalDeviceVulkan12Features available_features12{}; + available_features12.pNext = &available_features11; available_features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features; vk::PhysicalDeviceVulkan13Features available_features13{}; @@ -17191,6 +17223,10 @@ private: available_features.shaderImageGatherExtended == true && available_derivatives.computeDerivativeGroupQuads == true, "image gather or compute derivative quads are not supported"); + Require("VulkanHarness", "dispatch", + available_features11.storageBuffer16BitAccess == true && + available_features12.storageBuffer8BitAccess == true, + "8-bit and 16-bit storage buffer access are not supported"); Require("VulkanHarness", "dispatch", available_features12.timelineSemaphore == true, "timeline semaphores are not supported"); @@ -17240,11 +17276,13 @@ private: device_info.sType = vk::StructureType::eDeviceCreateInfo; device_info.queueCreateInfoCount = 1; device_info.pQueueCreateInfos = &queue_info; + auto device_features11 = WindowContext::RequiredVulkan11Features(); auto device_features12 = WindowContext::RequiredVulkan12Features(); device_features12.shaderSharedInt64Atomics = true; vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{}; workgroup_layout.workgroupMemoryExplicitLayout = true; - device_features12.pNext = &workgroup_layout; + device_features12.pNext = &device_features11; + device_features11.pNext = &workgroup_layout; vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{}; image_atomic64.shaderImageInt64Atomics = true; workgroup_layout.pNext = &image_atomic64; @@ -26186,7 +26224,7 @@ TestCase BufferStoreDwordAppliesHostOffset() { std::vector code; AppendVMovLiteral(&code, 0, 0xabcdef01u); - code.push_back(EncodeMubuf0(0x1cu, 0, false, false)); + code.push_back(EncodeMubuf0(0x1cu, 2, false, false)); code.push_back(EncodeMubuf1(0, 0, 20)); AppendEnd(&code); @@ -26194,11 +26232,11 @@ TestCase BufferStoreDwordAppliesHostOffset() { test.name = "BufferStoreDwordAppliesHostOffset"; test.code = code; test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0}; - test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u}; - test.storage_buffer_range_dwords = 1; - test.storage_buffer_offsets = {12}; + test.expected = {0x11111111u, 0xabcdef01u, 0x33333333u, 0}; + test.storage_buffer_range_bytes = 8; + test.storage_buffer_offsets = {2}; test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.user_data = MakeStructuredStorageBufferData(4, 1); + test.user_data = MakeStructuredStorageBufferData(0, 8); test.has_user_data = true; return test; } @@ -26226,7 +26264,7 @@ TestCase BufferOffsetsUsePackedLaneAndStorageFallback() { test.code = std::move(code); test.initial = {0, 0, 0, 0}; test.expected = {0x12345678u, 0, 0, 0xabcdef01u}; - test.storage_buffer_range_dwords = 1; + test.storage_buffer_range_bytes = 4; test.storage_buffer_offsets = {0, 12}; test.expand_shader_data_storage = true; test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD, @@ -26269,6 +26307,99 @@ TestCase BufferLoadVariants() { O::BUFFER_STORE_DWORD, O::S_ENDPGM}}; } +TestCase BufferSubwordLoadsAtHostOffset(u32 bytes, u32 component, bool sign) { + using O = ShaderOpcode; + TestCase test; + test.name = bytes == 1 ? (sign ? "BufferSbyteHostOffsetTail" : "BufferUbyteHostOffsetTail") + : sign ? "BufferSshortHostOffsetTail" + : component == 0 ? "BufferUshortHostOffsetFirst" + : component == 2 ? "BufferUshortHostOffsetCarry" + : "BufferUshortHostOffsetTail"; + // The captured stride-six index buffer uses USHORT IDXEN offsets 4, 2, 0. + test.code = {EncodeMubuf0((bytes == 1 ? 0x08u : 0x0au) + sign, + component, true, false), + EncodeMubuf1(1, 0, 0), + EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)}; + AppendEnd(&test.code); + test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu, + 0xccccccccu, 0xccccccccu, 0xccccccccu}; + test.expected = test.initial; + for (u32 record = 0; record < 2; ++record) { + u32 value = 0; + std::memcpy(&value, reinterpret_cast(test.initial.data()) + + bytes + record * 6 + component, bytes); + if (sign && (value & (1u << (bytes * 8u - 1u))) != 0) { + value |= UINT32_MAX << (bytes * 8u); + } + test.expected[4 + record] = value; + } + test.expected[6] = 0; + test.storage_buffer_range_bytes = 12; + test.storage_buffer_offsets = {bytes, 16}; + test.user_data = MakeStructuredStorageBufferData(6, 2); + test.user_data[0] = 0x1000u + bytes; + test.user_data[48] = 0x2000u; + test.user_data[49] = 4u << 16u; + test.user_data[50] = 3; + test.user_data[51] = 0x5204; + test.has_user_data = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.threads_num[0] = 3; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.has_compute_info = true; + test.opcodes = {bytes == 1 ? (sign ? O::BUFFER_LOAD_SBYTE : O::BUFFER_LOAD_UBYTE) + : (sign ? O::BUFFER_LOAD_SSHORT : O::BUFFER_LOAD_USHORT), + O::BUFFER_STORE_DWORD, O::S_ENDPGM}; + return test; +} + +TestCase BufferSubwordHostOffsetOverflow() { + auto test = BufferSubwordLoadsAtHostOffset(2, 0, false); + test.name = "BufferSubwordHostOffsetOverflow"; + test.code.clear(); + AppendVMovLiteral(&test.code, 20, UINT32_MAX); + test.code.insert(test.code.end(), + {EncodeMubuf0(0x0au), EncodeMubuf1(1, 0, 20), + EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)}); + AppendVMovLiteral(&test.code, 1, 0xbeef); + test.code.insert(test.code.end(), {EncodeMubuf0(0x1au), EncodeMubuf1(1, 0, 20)}); + AppendEnd(&test.code); + test.expected[4] = test.expected[5] = 0; + test.opcodes.push_back(ShaderOpcode::V_MOV_B32); + test.opcodes.push_back(ShaderOpcode::BUFFER_STORE_SHORT); + return test; +} + +TestCase BufferSubwordStoresAtHostOffset(u32 bytes) { + using O = ShaderOpcode; + TestCase test; + test.name = bytes == 1 ? "BufferByteStoreHostOffsetTail" : "BufferShortStoreHostOffsetTail"; + AppendVMovLiteral(&test.code, 1, bytes == 1 ? 0xa5u : 0xcafeu); + test.code.push_back(EncodeMubuf0(bytes == 1 ? 0x18u : 0x1au, + 6u - bytes, true, false)); + test.code.push_back(EncodeMubuf1(1, 0, 0)); + AppendEnd(&test.code); + test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu, + 0x13579bdfu, 0x2468ace0u}; + test.expected = bytes == 1 + ? std::vector{0x800112abu, 0xfea52345u, 0x80007fffu, 0xfaceffa5u, + 0x13579bdfu, 0x2468ace0u} + : std::vector{0x800112abu, 0xcafe2345u, 0x80007fffu, 0xfacecafeu, + 0x13579bdfu, 0x2468ace0u}; + test.storage_buffer_range_bytes = 12; + test.storage_buffer_offsets = {bytes}; + test.user_data = MakeStructuredStorageBufferData(6, 2); + test.user_data[0] = 0x1000u + bytes; + test.has_user_data = true; + test.compute_info.thread_ids_num = 1; + test.compute_info.threads_num[0] = 3; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.has_compute_info = true; + test.opcodes = {O::V_MOV_B32, bytes == 1 ? O::BUFFER_STORE_BYTE : O::BUFFER_STORE_SHORT, + O::S_ENDPGM}; + return test; +} + TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() { using O = ShaderOpcode; @@ -26363,7 +26494,7 @@ TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = {0x33333333u, 0x44444444u, 0u, 0u}; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; return test; @@ -26574,7 +26705,7 @@ TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu}; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORDX4, O::S_ENDPGM}; return test; } @@ -26595,7 +26726,7 @@ TestCase BufferLoadFormatXyzwRejectsPartialRecord() { test.code = std::move(code); test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u}; test.expected = {0u, 0u, 0u, 0u}; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.user_data = MakeStructuredStorageBufferData( 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -26622,7 +26753,7 @@ TestCase BufferStoreFormatXyzwDropsPartialRecord() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = test.initial; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.user_data = MakeStructuredStorageBufferData( 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -26648,7 +26779,7 @@ TestCase BufferLoadFormatXRejectsPartialRecord() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = {0u, 0u, 0x33333333u, 0x44444444u}; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.user_data = MakeStructuredStorageBufferData( 0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -26672,7 +26803,7 @@ TestCase BufferStoreFormatXRejectsPartialRecord() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = test.initial; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.user_data = MakeStructuredStorageBufferData( 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -26695,7 +26826,7 @@ TestCase BufferLoadFormatXyRejectsPartialRecord() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = {0u, 0u, 0x33333333u, 0x44444444u}; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.user_data = MakeStructuredStorageBufferData( 0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -26720,7 +26851,7 @@ TestCase BufferStoreFormatXyRejectsPartialRecord() { test.code = std::move(code); test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; test.expected = test.initial; - test.storage_buffer_range_dwords = 4; + test.storage_buffer_range_bytes = 16; test.user_data = MakeStructuredStorageBufferData( 0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float)); test.has_user_data = true; @@ -27030,7 +27161,7 @@ TestCase BufferStoreFormatXResource16UintPreservesAdjacentLanes() { test.has_compute_info = true; test.user_data = MakeStructuredStorageBufferData(2, 64, false, 11); test.has_user_data = true; - test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"}; + test.forbidden_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"}; return test; } @@ -27383,7 +27514,7 @@ TestCase BufferStoreFormatXDropsOutOfRangeRecord() { test.code = code; test.initial = {0x11111111u, 0x22222222u}; test.expected = {0x11111111u, 0x22222222u}; - test.storage_buffer_range_dwords = 1; + test.storage_buffer_range_bytes = 4; test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM}; test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20); test.has_user_data = true; @@ -31534,7 +31665,7 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) { TestCase test; test.name = "IndirectBufferStore"; test.initial.assign(64, 0xdeadbeefu); - test.storage_buffer_range_dwords = 16; + test.storage_buffer_range_bytes = 64; test.storage_buffer_offsets = {12, 108}; test.required_spirv = {"OpSwitch"}; test.forbidden_spirv = {"PhysicalStorageBuffer"}; @@ -33303,6 +33434,15 @@ std::vector MakeCases() { AddCase(BufferStoreDwordAppliesHostOffset); AddCase(BufferOffsetsUsePackedLaneAndStorageFallback); AddCase(BufferLoadVariants); + for (const u32 component : {4u, 2u, 0u}) { + cases.push_back(BufferSubwordLoadsAtHostOffset(2, component, false)); + } + cases.push_back(BufferSubwordLoadsAtHostOffset(2, 4, true)); + cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, false)); + cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, true)); + AddCase(BufferSubwordHostOffsetOverflow); + cases.push_back(BufferSubwordStoresAtHostOffset(2)); + cases.push_back(BufferSubwordStoresAtHostOffset(1)); AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress); AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress); AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);