diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp index 0a39f9b3c..b3c96be36 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp @@ -141,7 +141,8 @@ uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mi uint32_t array_index) { const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index); const auto image = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpLoad, ImageType(state, state.program.info.images.at(resource)), + const auto kind = *IR::DescriptorBindingForImage(state.program.info.images[resource]); + state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type, image, pointer); return image; } @@ -151,14 +152,11 @@ uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Samplers, sampler, "sampler resource index is out of range"); } - const auto sampler_type = state.builder.Type(spv::OpTypeSampler); - const auto pointer_type = - state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type); const auto pointer = DescriptorElementPointer( - state, pointer_type, state.sampler_variable, ConstantU32(state, sampler), + state, state.sampler_pointer_type, state.sampler_variable, ConstantU32(state, sampler), IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); const auto sampler_id = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer); + state.builder.AddFunction(spv::OpLoad, state.sampler_type, sampler_id, pointer); return sampler_id; } @@ -168,8 +166,12 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled); const auto image = LoadImageDescriptor(state, resource, mip, array_index); const auto sampled_image = state.builder.AllocateId(); - const auto sampled_type = - state.builder.Type(spv::OpTypeSampledImage, ImageType(state, image_resource)); + const auto kind = *IR::DescriptorBindingForImage(image_resource); + auto& sampled_type = state.images[IR::ImageBindingIndex(kind)].sampled_type; + if (sampled_type == 0) { + sampled_type = state.builder.Type( + spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type); + } state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id); if (array_index != 0u) { state.builder.RequireExtension("SPV_EXT_descriptor_indexing"); @@ -189,10 +191,9 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t if (array_index == 0u) { array_index = ConstantU32(state, image.descriptor_index + mip); } - const auto pointer_type = state.builder.Type( - spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image)); - const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)]; - return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource, + const auto& definition = state.images[IR::ImageBindingIndex(*kind)]; + return DescriptorElementPointer(state, definition.pointer_type, definition.variable, + array_index, *kind, resource, "image descriptor array was not emitted"); } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 2f5df6ebd..ba02bf0fb 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -57,6 +57,21 @@ constexpr std::array ImageDimensions {{ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); +struct BufferDefinition { + uint32_t variable = 0; + uint32_t type = 0; + uint32_t element_type = 0; + uint32_t pointer_type = 0; + uint32_t element_pointer_type = 0; +}; + +struct ImageDefinition { + uint32_t variable = 0; + uint32_t type = 0; + uint32_t pointer_type = 0; + uint32_t sampled_type = 0; +}; + struct EmitterState { EmitterState(IR::Program& program_, ShaderStageInputInfo input_info_) : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u), @@ -93,10 +108,7 @@ struct EmitterState { uint32_t lane_count = 1; uint32_t lane_half = 0; - uint32_t storage_buffer_variable = 0; - uint32_t storage_buffer_u8_variable = 0; - uint32_t storage_buffer_u16_variable = 0; - uint32_t storage_buffer_u64_variable = 0; + std::array storage_buffers {}; std::array memory_byte_offsets {}; uint32_t bda_pagetable_variable = 0; uint32_t fault_buffer_variable = 0; @@ -112,8 +124,10 @@ struct EmitterState { uint32_t lds_variable = 0; uint32_t lds_u64_variable = 0; std::array scratch_variable {}; - std::array image_variables {}; + std::array images {}; uint32_t sampler_variable = 0; + uint32_t sampler_type = 0; + uint32_t sampler_pointer_type = 0; uint32_t main_func = 0; uint32_t mesh_guest_func = 0; uint32_t mesh_allocation = 0; @@ -156,8 +170,6 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components); uint32_t TypeF32Vector(EmitterState& state, uint32_t components); uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee); uint32_t TypeFunction(EmitterState& state); -uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits); -uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits = 32); uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32); uint32_t TypePhysicalU32Pointer(EmitterState& state); uint32_t TypePushConstantElementPointer(EmitterState& state); @@ -405,7 +417,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t variable, uint32_t pointer_type); + BufferDefinition& buffer); uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, uint32_t index); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index ed742a57e..d596c63b7 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -332,7 +332,8 @@ uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& uint32_t value; if (resource.element_bits < 32u) { const auto loaded = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction(spv::OpLoad, TypeStorageBufferElement(ctx.state, bits), loaded, + ctx.state.builder.AddFunction(spv::OpLoad, + ctx.state.storage_buffers[bits == 8u ? 0 : 1].element_type, loaded, EmitMemoryElementPointer(ctx.state, resource, index), resource.memory_access); value = Unary(ctx.state, spv::OpUConvert, TypeU32(ctx.state), loaded); @@ -437,7 +438,8 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, if (resource.element_bits < 32u) { ctx.state.builder.AddFunction( spv::OpStore, pointer, - Unary(ctx.state, spv::OpUConvert, TypeStorageBufferElement(ctx.state, bits), data), + Unary(ctx.state, spv::OpUConvert, + ctx.state.storage_buffers[bits == 8u ? 0 : 1].element_type, data), resource.memory_access); return; } @@ -1199,8 +1201,8 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition( state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() { - const auto resource = PrepareStorageBufferResourceAccess( - state, mem, state.storage_buffer_u64_variable, TypeStorageBufferPointer(state, 64)); + const auto resource = + PrepareStorageBufferResourceAccess(state, mem, state.storage_buffers[3]); const auto byte_address = ByteAddress(ctx, inst, mem); const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address, ConstantU32(state, 3u)); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 60d1292e2..1b990d182 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -152,20 +152,22 @@ void EnsureLdsStorage(EmitterState& state) { MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t variable, - uint32_t pointer_type) { - if (variable == 0) { + BufferDefinition& buffer) { + if (buffer.variable == 0) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "storage buffer descriptor array was not emitted"); } + if (buffer.pointer_type == 0) { + buffer.pointer_type = TypePointer(state, spv::StorageClassStorageBuffer, buffer.type); + } const auto array_index = state.program.info.buffers.at(mem.resource).descriptor_index; MemoryResourceAccess access { .kind = mem.kind, .memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone}; access.object_pointer = state.builder.AllocateId(); - state.builder.AddFunction(spv::OpAccessChain, pointer_type, access.object_pointer, variable, - ConstantU32(state, array_index)); + state.builder.AddFunction(spv::OpAccessChain, buffer.pointer_type, access.object_pointer, + buffer.variable, ConstantU32(state, array_index)); access.byte_offset = state.memory_byte_offsets[array_index]; access.length = state.builder.AllocateId(); state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, @@ -207,11 +209,13 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR:: case IR::ResourceKind::Buffer: { const auto bits = mem.kind == IR::ResourceKind::Buffer ? IR::StorageBufferElementBits(state.program, mem) : 32u; - const auto variable = bits == 8u ? state.storage_buffer_u8_variable - : bits == 16u ? state.storage_buffer_u16_variable - : state.storage_buffer_variable; - access = PrepareStorageBufferResourceAccess( - state, mem, variable, TypeStorageBufferPointer(state, bits)); + auto* buffer = &state.storage_buffers[2]; + if (bits == 8u) { + buffer = &state.storage_buffers[0]; + } else if (bits == 16u) { + buffer = &state.storage_buffers[1]; + } + access = PrepareStorageBufferResourceAccess(state, mem, *buffer); access.element_bits = bits; return access; } diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index e30044c97..8ccd09d22 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -120,32 +120,39 @@ uint32_t TypeFunction(EmitterState& state) { return state.function_type; } -uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) { - return bits == 32u ? TypeU32(state) - : bits == 64u ? TypeU64(state) - : state.builder.Type(spv::OpTypeInt, bits, 0); -} - -uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) { - return state.builder.DecoratedType( - spv::OpTypeRuntimeArray, - {{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}}, - TypeStorageBufferElement(state, bits)); -} - -uint32_t StorageBufferType(EmitterState& state, uint32_t bits = 32) { - return state.builder.DecoratedType(spv::OpTypeStruct, - {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, - {spv::OpDecorate, {spv::DecorationBlock}}}, - StorageRuntimeArrayType(state, bits)); -} - -uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits) { - return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state, bits)); +uint32_t StorageBufferType(EmitterState& state, BufferDefinition& buffer, uint32_t bits = 32) { + if (buffer.type == 0) { + if (bits == 32u) { + buffer.element_type = TypeU32(state); + } else if (bits == 64u) { + buffer.element_type = TypeU64(state); + } else { + buffer.element_type = state.builder.Type(spv::OpTypeInt, bits, 0); + } + const auto array = state.builder.DecoratedType( + spv::OpTypeRuntimeArray, + {{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}}, buffer.element_type); + buffer.type = state.builder.DecoratedType( + spv::OpTypeStruct, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, + {spv::OpDecorate, {spv::DecorationBlock}}}, array); + } + return buffer.type; } uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) { - return TypePointer(state, spv::StorageClassStorageBuffer, TypeStorageBufferElement(state, bits)); + auto* buffer = &state.storage_buffers[2]; + if (bits == 8u) { + buffer = &state.storage_buffers[0]; + } else if (bits == 16u) { + buffer = &state.storage_buffers[1]; + } else if (bits == 64u) { + buffer = &state.storage_buffers[3]; + } + if (buffer->element_pointer_type == 0) { + buffer->element_pointer_type = + TypePointer(state, spv::StorageClassStorageBuffer, buffer->element_type); + } + return buffer->element_pointer_type; } uint32_t TypePhysicalU32Pointer(EmitterState& state) { @@ -264,60 +271,57 @@ void DefineDescriptors(EmitterState& state) { ConstantU32(state, counts[index])); }; switch (kind) { - case IR::DescriptorBindingKind::Buffers: - state.storage_buffer_variable = - Define(ArrayType(StorageBufferType(state)), "buffers"); - if (state.program.info.buffer_u8) { - state.storage_buffer_u8_variable = - Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8"); - } - if (state.program.info.buffer_u16) { - state.storage_buffer_u16_variable = - Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16"); - } - if (state.program.info.buffer_int64_atomics) { - state.storage_buffer_u64_variable = - Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64"); - } - for (const auto variable: {state.storage_buffer_variable, state.storage_buffer_u8_variable, - state.storage_buffer_u16_variable, state.storage_buffer_u64_variable}) { - if (variable == 0) continue; - if (state.storage_buffer_u8_variable != 0 || state.storage_buffer_u16_variable != 0 || - state.storage_buffer_u64_variable != 0) { - state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); + case IR::DescriptorBindingKind::Buffers: { + const std::array used {info.buffer_u8, info.buffer_u16, true, info.buffer_int64_atomics}; + constexpr std::array names {"buffers_u8", "buffers_u16", "buffers", "buffers_u64"}; + for (uint32_t alias = 0; alias < used.size(); ++alias) { + if (!used[alias]) continue; + auto& buffer = state.storage_buffers[alias]; + const auto type = StorageBufferType(state, buffer, 8u << alias); + buffer.variable = Define(ArrayType(type), names[alias]); + if (info.buffer_u8 || info.buffer_u16 || info.buffer_int64_atomics) { + state.builder.AddAnnotation(spv::OpDecorate, buffer.variable, spv::DecorationAliased); } // RDNA2 stores publish to L2 even without GLC; every alias of the buffer // must participate in visibility for cache-bypassing polling loads. - if (state.program.info.coherent_buffers) { - state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent); + if (info.coherent_buffers) { + state.builder.AddAnnotation(spv::OpDecorate, buffer.variable, spv::DecorationCoherent); } } break; + } case IR::DescriptorBindingKind::BdaPagetable: - state.bda_pagetable_variable = Define(StorageBufferType(state, 64), "bda_pagetable"); + state.bda_pagetable_variable = + Define(StorageBufferType(state, state.storage_buffers[3], 64), "bda_pagetable"); break; case IR::DescriptorBindingKind::FaultBuffer: - state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer"); + state.fault_buffer_variable = + Define(StorageBufferType(state, state.storage_buffers[2]), "fault_buffer"); break; case IR::DescriptorBindingKind::ShaderData: state.shader_data_storage_variable = - Define(StorageBufferType(state), "shader_data"); + Define(StorageBufferType(state, state.storage_buffers[2]), "shader_data"); break; case IR::DescriptorBindingKind::FlattenedSrt: - state.flattened_srt_variable = Define(StorageBufferType(state), "flattened_srt"); + state.flattened_srt_variable = + Define(StorageBufferType(state, state.storage_buffers[2]), "flattened_srt"); break; case IR::DescriptorBindingKind::Samplers: - state.sampler_variable = Define(ArrayType(state.builder.Type(spv::OpTypeSampler)), - "samplers", spv::StorageClassUniformConstant); + state.sampler_type = state.builder.Type(spv::OpTypeSampler); + state.sampler_pointer_type = TypePointer( + state, spv::StorageClassUniformConstant, state.sampler_type); + state.sampler_variable = Define(ArrayType(state.sampler_type), "samplers", + spv::StorageClassUniformConstant); break; case IR::DescriptorBindingKind::Gds: - state.gds_variable = Define(StorageBufferType(state), "gds"); + state.gds_variable = Define(StorageBufferType(state, state.storage_buffers[2]), "gds"); break; case IR::DescriptorBindingKind::SharedMemory: - state.lds_variable = Define(StorageBufferType(state), "lds_dwords"); + state.lds_variable = Define(StorageBufferType(state, state.storage_buffers[2]), "lds_dwords"); state.builder.AddAnnotation(spv::OpDecorate, state.lds_variable, spv::DecorationCoherent); if (state.program.info.shared_int64_atomics) { - state.lds_u64_variable = Define(StorageBufferType(state, 64), "lds_qwords"); + state.lds_u64_variable = + Define(StorageBufferType(state, state.storage_buffers[3], 64), "lds_qwords"); state.builder.AddAnnotation(spv::OpDecorate, state.lds_u64_variable, spv::DecorationCoherent); for (const auto variable: {state.lds_variable, state.lds_u64_variable}) { state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); @@ -329,9 +333,12 @@ void DefineDescriptors(EmitterState& state) { IR::ImageResourceClass::None); const auto& image = *image_types[IR::ImageBindingIndex(kind)]; const auto name = "image_" + std::to_string(static_cast(kind)); - state.image_variables[IR::ImageBindingIndex(kind)] = - Define(ArrayType(ImageType(state, image)), name.c_str(), - spv::StorageClassUniformConstant); + auto& definition = state.images[IR::ImageBindingIndex(kind)]; + definition.type = ImageType(state, image); + definition.variable = Define(ArrayType(definition.type), name.c_str(), + spv::StorageClassUniformConstant); + definition.pointer_type = TypePointer( + state, spv::StorageClassUniformConstant, definition.type); if (image.dimension == ImageDimension::Dim1D || image.dimension == ImageDimension::Dim1DArray) { state.builder.RequireCapability(image.resource_class ==