From 3ea20992459a4ff44eed1f429e26e0bd0f73c308 Mon Sep 17 00:00:00 2001 From: nmzik Date: Wed, 19 Aug 2026 06:32:47 +0200 Subject: [PATCH] refactor(shader): simplify buffer access emission --- .../shader/recompiler/ShaderRecompiler.cpp | 4 +- .../recompiler/backend/spirv/SpirvEmitter.cpp | 62 +-- .../recompiler/backend/spirv/SpirvEmitter.h | 2 +- .../backend/spirv/spirvEmitterHelpers.cpp | 2 +- .../backend/spirv/spirvEmitterInternal.h | 49 +- .../backend/spirv/spirvEmitterMemory.cpp | 417 +++++++++--------- .../spirv/spirvEmitterMemoryHelpers.cpp | 204 ++++----- .../recompiler/ir/opcodes/ValueOpcodes.cpp | 44 +- .../recompiler/ir/opcodes/ValueOpcodes.h | 3 + .../recompiler/ir/passes/ResourceTracking.cpp | 52 +-- tests/ShaderRecompilerComputeTests.cpp | 20 +- tests/shaderCfgTests.cpp | 89 +++- 12 files changed, 474 insertions(+), 474 deletions(-) diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index d346f5548..8cd053e93 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -942,7 +942,9 @@ bool TryRecompile(std::span code, const CompileOptions& options, if (!IR::AllocateBindings(ir, layout_options, error)) { return false; } - Spirv::AnalyzeProgramRequirements(ir); + if (!Spirv::AnalyzeProgramRequirements(ir, error)) { + return false; + } std::string ir_dump; if (options.dump_ir) { ir_dump = MakeIrDump(cfg, ir); diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index 939eaea90..bc4134e26 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -6,8 +6,6 @@ #include #include -#include -#include namespace Libs::Graphics::ShaderRecompiler::Spirv { @@ -219,23 +217,12 @@ bool ValidateNativeProgram(const IR::Program& program, std::string* error) { } // namespace -static constexpr size_t InitialEmitterVectorReserve = 4096; - -static void ReportReserveExceeded(const IR::Program& program, const char* vector_name, - size_t size) { - if (size <= InitialEmitterVectorReserve) { - return; - } - std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64 - " stage=%u vector=%s size=%zu reserve=%zu\n", - program.shader_hash, static_cast(program.stage), vector_name, size, - InitialEmitterVectorReserve); -} - -void AnalyzeProgramRequirements(IR::Program& program) { - auto& requirements = program.spirv_requirements.emplace(); +bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) { + program.spirv_requirements.reset(); + IR::SpirvRequirements requirements {}; if (program.values == nullptr) { - return; + program.spirv_requirements.emplace(requirements); + return true; } const auto MarkExactSubgroup = [&] { requirements.requires_exact_subgroup = true; @@ -243,6 +230,25 @@ void AnalyzeProgramRequirements(IR::Program& program) { }; for (const auto* block: program.values->blocks) { for (const auto& inst: *block) { + if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { + const auto memory_index = inst.Flags().index; + if (memory_index >= program.values->memory_info.size()) { + return Fail(error, "buffer operation has invalid memory metadata"); + } + const auto& memory = program.values->memory_info[memory_index]; + if (memory.kind == IR::ResourceKind::Buffer) { + if (memory.resource >= program.info.buffers.size()) { + return Fail(error, "buffer operation has invalid resource metadata"); + } + if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { + if (program.stage != ShaderType::Compute) { + return Fail(error, "buffer ADD_TID is only valid for compute shaders"); + } + requirements.requires_exact_subgroup = true; + requirements.subgroup_local_invocation_id = true; + } + } + } switch (inst.GetOpcode()) { case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break; case IR::ValueOpcode::DppMoveU32: @@ -302,8 +308,10 @@ void AnalyzeProgramRequirements(IR::Program& program) { case IR::ValueOpcode::SharedAtomicOr32: case IR::ValueOpcode::SharedAtomicXor32: { const auto index = inst.Flags().index; + if (index >= program.values->memory_info.size()) { + return Fail(error, "shared operation has invalid memory metadata"); + } if (program.stage != ShaderType::Compute && - index < program.values->memory_info.size() && program.values->memory_info[index].kind == IR::ResourceKind::Lds) { requirements.function_lds = true; } @@ -318,8 +326,10 @@ void AnalyzeProgramRequirements(IR::Program& program) { break; case IR::ValueOpcode::SetAttribute: { const auto index = inst.Flags().index; + if (index >= program.values->export_info.size()) { + return Fail(error, "attribute export has invalid metadata"); + } if (program.stage == ShaderType::Pixel && - index < program.values->export_info.size() && program.values->export_info[index].vm) { requirements.pixel_valid_mask = true; } @@ -329,6 +339,8 @@ void AnalyzeProgramRequirements(IR::Program& program) { } } } + program.spirv_requirements.emplace(requirements); + return true; } bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, @@ -368,9 +380,9 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc state.stage = program.stage; state.wave_size = program.wave_size; state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation; - state.inputs.reserve(InitialEmitterVectorReserve); - state.outputs.reserve(InitialEmitterVectorReserve); - state.interface_variables.reserve(InitialEmitterVectorReserve); + state.inputs.reserve(program.info.inputs.size()); + state.outputs.reserve(program.info.outputs.size()); + state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size()); CopyProgramInputsAndOutputs(state, program); AllocateInputVariables(state); AllocateOutputVariables(state); @@ -379,10 +391,6 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc return false; } - ReportReserveExceeded(program, "inputs", state.inputs.size()); - ReportReserveExceeded(program, "outputs", state.outputs.size()); - ReportReserveExceeded(program, "interface_variables", state.interface_variables.size()); - auto binary = state.builder.Build(); if (binary.empty()) { SetError(error, "SPIR-V builder returned an empty module"); diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h index d5ce807e8..18db206ff 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h @@ -9,7 +9,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv { -void AnalyzeProgramRequirements(IR::Program& program); +bool AnalyzeProgramRequirements(IR::Program& program, std::string* error); bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources, ShaderStageInputInfo input_info, std::vector& spirv, std::string* error); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp index 11ebc8c5f..ac39454cf 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp @@ -115,7 +115,7 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) { uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) { if (state.subgroup_local_invocation_id_variable == 0) { - return ConstantU32(state, 0); + EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n"); } const auto value = state.builder.AllocateId(); state.builder.AddFunction( diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 78bea4526..1f442f41e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -672,38 +672,32 @@ uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint3 uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index); -uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc); +uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem); -Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc); +Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem); void EmitStorageBufferOffsets(EmitterState& state); -DescriptorResourceBinding StorageBufferBindingForMemory(EmitterState& state, - const IR::MemoryInfo& mem, uint32_t use_pc); - -uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc); - -uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t index, uint32_t use_pc); - -uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t index, uint32_t use_pc); - -uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index); - uint32_t LdsDwordCount(const EmitterState& state); -uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index); +struct MemoryResourceAccess { + IR::ResourceKind kind = IR::ResourceKind::None; + uint32_t object_pointer = 0; + uint32_t length = 0; + uint32_t index_offset = 0; + bool add_index_offset = false; +}; -uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index); +MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem); -uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index); +uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access, + uint32_t raw_index); -uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, - uint32_t use_pc); +uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, + uint32_t index); + +uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access, + uint32_t index); uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value); @@ -807,11 +801,6 @@ bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std: // These templates accept local lambdas from several emitter translation units. template void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { - if (condition == 0) { - fn(); - return; - } - const auto then_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId(); state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); @@ -825,10 +814,6 @@ void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { template uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type, uint32_t default_value, Fn&& fn) { - if (condition == 0) { - return fn(); - } - const auto then_label = state.builder.AllocateId(); const auto then_exit = state.builder.AllocateId(); const auto else_label = state.builder.AllocateId(); diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 71907ea55..4aa43caa1 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -25,68 +25,63 @@ uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t } uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) { - if (lhs == 0) return rhs; - if (rhs == 0) return lhs; return Binary(state, OpLogicalAnd, TypeBool(state), lhs, rhs); } uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, uint32_t index, uint32_t offset, uint32_t soffset) { - auto& state = ctx.state; - const auto packed = - ConstantU32(state, StorageBufferPackedStride(state, mem, inst.Flags().pc)); - if (mem.formatted && !mem.typed && inst.GetOpcode() == IR::ValueOpcode::StoreBufferU32) { - const auto add_tid_bit = Binary( - state, OpBitwiseAnd, TypeU32(state), - Binary(state, OpShiftRightLogical, TypeU32(state), packed, ConstantU32(state, 20)), - ConstantU32(state, 1)); - const auto add_tid = - Binary(state, OpINotEqual, TypeBool(state), add_tid_bit, ConstantU32(state, 0)); + auto& state = ctx.state; + const uint32_t packed = StorageBufferPackedStride(state, mem); + const uint32_t stride = packed & 0x3fffu; + const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u; + if (((packed >> 20u) & 1u) != 0u) { const auto lane = Binary(state, OpBitwiseAnd, TypeU32(state), - EmitLocalInvocationIndex(state), ConstantU32(state, 63)); - index = Select(state, TypeU32(state), add_tid, - Binary(state, OpIAdd, TypeU32(state), index, lane), index); + EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63)); + index = Binary(state, OpIAdd, TypeU32(state), index, lane); + } + if (mem.offset != 0u) { + offset = Binary(state, OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset)); + } + + uint32_t address = 0; + if (!swizzle) { + if (stride == 0u) { + address = offset; + } else { + const auto indexed = stride == 1u ? index + : Binary(state, OpIMul, TypeU32(state), index, + ConstantU32(state, stride)); + address = Binary(state, OpIAdd, TypeU32(state), indexed, offset); + } + } else { + const uint32_t stride_enum = (packed >> 16u) & 3u; + const uint32_t index_stride = 8u << stride_enum; + const auto index_msb = Binary(state, OpShiftRightLogical, TypeU32(state), index, + ConstantU32(state, stride_enum + 3u)); + const auto index_lsb = Binary(state, OpBitwiseAnd, TypeU32(state), index, + ConstantU32(state, index_stride - 1u)); + const auto offset_msb = + Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u)); + const auto offset_lsb = + Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u)); + const auto indexed_msb = stride == 1u ? index_msb + : Binary(state, OpIMul, TypeU32(state), index_msb, + ConstantU32(state, stride)); + const auto msb = Binary(state, OpIMul, TypeU32(state), + Binary(state, OpIAdd, TypeU32(state), indexed_msb, offset_msb), + ConstantU32(state, index_stride)); + const auto lsb = Binary( + state, OpIAdd, TypeU32(state), + Binary(state, OpShiftLeftLogical, TypeU32(state), index_lsb, ConstantU32(state, 2u)), + offset_lsb); + address = Binary(state, OpIAdd, TypeU32(state), msb, lsb); + } + + const auto soffset_value = inst.Arg(3).Resolve(); + if (soffset_value.IsImmediate() && soffset_value.GetType() == IR::Type::U32 && + soffset_value.U32() == 0u) { + return address; } - const auto stride = - Binary(state, OpBitwiseAnd, TypeU32(state), packed, ConstantU32(state, 0x3fffu)); - const auto swizzle = - Binary(state, OpBitwiseAnd, TypeU32(state), - Binary(state, OpShiftRightLogical, TypeU32(state), packed, ConstantU32(state, 14)), - ConstantU32(state, 1)); - const auto use_swizzle = - Binary(state, OpLogicalAnd, TypeBool(state), - Binary(state, OpINotEqual, TypeBool(state), stride, ConstantU32(state, 0)), - Binary(state, OpINotEqual, TypeBool(state), swizzle, ConstantU32(state, 0))); - const auto stride_enum = - Binary(state, OpBitwiseAnd, TypeU32(state), - Binary(state, OpShiftRightLogical, TypeU32(state), packed, ConstantU32(state, 16)), - ConstantU32(state, 3)); - const auto index_shift = - Binary(state, OpIAdd, TypeU32(state), stride_enum, ConstantU32(state, 3)); - const auto index_stride = - Binary(state, OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 8), stride_enum); - const auto index_mask = - Binary(state, OpISub, TypeU32(state), index_stride, ConstantU32(state, 1)); - offset = Binary(state, OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset)); - const auto linear = Binary(state, OpIAdd, TypeU32(state), - Binary(state, OpIMul, TypeU32(state), index, stride), offset); - const auto index_msb = Binary(state, OpShiftRightLogical, TypeU32(state), index, index_shift); - const auto index_lsb = Binary(state, OpBitwiseAnd, TypeU32(state), index, index_mask); - const auto offset_msb = - Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u)); - const auto offset_lsb = - Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3)); - const auto msb = - Binary(state, OpIMul, TypeU32(state), - Binary(state, OpIAdd, TypeU32(state), - Binary(state, OpIMul, TypeU32(state), index_msb, stride), offset_msb), - index_stride); - const auto lsb = - Binary(state, OpIAdd, TypeU32(state), - Binary(state, OpShiftLeftLogical, TypeU32(state), index_lsb, ConstantU32(state, 2)), - offset_lsb); - const auto address = Select(state, TypeU32(state), use_swizzle, - Binary(state, OpIAdd, TypeU32(state), msb, lsb), linear); return Binary(state, OpIAdd, TypeU32(state), address, soffset); } @@ -156,68 +151,66 @@ uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor ConstantU32(ctx.state, 2)); } -uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) { - if (mem.kind == IR::ResourceKind::Lds) { - return EmitLdsElementInBounds(ctx.state, index); - } - if (mem.kind == IR::ResourceKind::Gds) { - return EmitGdsElementInBounds(ctx.state, index); - } - return EmitStorageBufferElementInBounds(ctx.state, mem, index, pc); +struct PreparedMemoryElement { + MemoryResourceAccess resource; + uint32_t index = 0; +}; + +PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem, + uint32_t raw_index) { + auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index); + return {.resource = resource, .index = index}; } -uint32_t Pointer(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) { - return EmitMemoryElementPointer(ctx.state, mem, index, pc); -} - -uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, - uint32_t extra_offset = 0) { - mem.offset += extra_offset; - const auto pc = inst.Flags().pc; +uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto index = DwordIndex(ctx, inst, mem); - return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - const auto value = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {OpLoad, TypeU32(ctx.state), value, Pointer(ctx, mem, index, pc)}); - return value; - }); + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + const auto value = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, TypeU32(ctx.state), value, + EmitMemoryElementPointer(ctx.state, access.resource, access.index)}); + return value; + }); }); } uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits, - bool sign_extend, uint32_t extra_offset = 0) { - mem.offset += extra_offset; - const auto pc = inst.Flags().pc; + bool sign_extend) { return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto address = ByteAddress(ctx, inst, mem); - const auto index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 2)); - return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - const auto word = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {OpLoad, TypeU32(ctx.state), word, Pointer(ctx, mem, index, pc)}); - const auto byte = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 3)); - const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte, - ConstantU32(ctx.state, 3)); - const auto value = - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), - Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); - if (!sign_extend) return value; - const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value, - ConstantU32(ctx.state, 32u - bits)); - return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left, - ConstantU32(ctx.state, 32u - bits)); - }); + const auto address = ByteAddress(ctx, inst, mem); + const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 2)); + const auto access = PrepareMemoryElement(ctx, mem, raw_index); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + const auto word = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, TypeU32(ctx.state), word, + EmitMemoryElementPointer(ctx.state, access.resource, access.index)}); + const auto byte = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 3)); + const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte, + ConstantU32(ctx.state, 3)); + const auto value = + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), + Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); + if (!sign_extend) return value; + const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value, + ConstantU32(ctx.state, 32u - bits)); + return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left, + ConstantU32(ctx.state, 32u - bits)); + }); }); } Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { return mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format) - : StorageBufferFormat(ctx.state, mem, inst.Flags().pc); + : StorageBufferFormat(ctx.state, mem); } IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format, @@ -311,54 +304,55 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki return observed; } -void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits, - uint32_t extra_offset = 0) { - mem.offset += extra_offset; - const auto pc = inst.Flags().pc; +void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto address = ByteAddress(ctx, inst, mem); - const auto index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 2)); - EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - const auto pointer = Pointer(ctx, mem, index, pc); - const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, - ConstantU32(ctx.state, 3)), - ConstantU32(ctx.state, 3)); - const auto mask = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift); - const auto value = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), - ctx.Arg(inst, inst.NumArgs() - 2), - ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)), - shift); - const auto merge = [&](uint32_t old) { - return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), - Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old, - Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)), - value); - }; - if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { - AtomicUpdate(ctx.state, pointer, mem.kind, merge); - } else { - const auto old = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer}); - ctx.state.builder.AddFunction({OpStore, pointer, merge(old)}); - } - }); + const auto address = ByteAddress(ctx, inst, mem); + const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, + ConstantU32(ctx.state, 2)); + const auto access = PrepareMemoryElement(ctx, mem, raw_index); + EmitIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + const auto pointer = + EmitMemoryElementPointer(ctx.state, access.resource, access.index); + const auto shift = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), + address, ConstantU32(ctx.state, 3)), + ConstantU32(ctx.state, 3)); + const auto mask = + Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift); + const auto value = + Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), + ctx.Arg(inst, inst.NumArgs() - 2), + ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)), + shift); + const auto merge = [&](uint32_t old) { + return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), + Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old, + Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)), + value); + }; + if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) { + AtomicUpdate(ctx.state, pointer, mem.kind, merge); + } else { + const auto old = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer}); + ctx.state.builder.AddFunction({OpStore, pointer, merge(old)}); + } + }); }); } -void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, - uint32_t extra_offset = 0) { - mem.offset += extra_offset; - const auto pc = inst.Flags().pc; +void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto index = DwordIndex(ctx, inst, mem); - EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - ctx.state.builder.AddFunction( - {OpStore, Pointer(ctx, mem, index, pc), ctx.Arg(inst, inst.NumArgs() - 2)}); - }); + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + EmitIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + ctx.state.builder.AddFunction( + {OpStore, EmitMemoryElementPointer(ctx.state, access.resource, access.index), + ctx.Arg(inst, inst.NumArgs() - 2)}); + }); }); } @@ -417,29 +411,32 @@ uint32_t AtomicOpcode(IR::ValueOpcode opcode) { uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, bool return_value) { - const auto pc = inst.Flags().pc; const auto result = EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto index = DwordIndex(ctx, inst, mem); - return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - const auto value = ctx.Arg(inst, inst.NumArgs() - 2); - const auto old = ctx.state.builder.AllocateId(); - const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; - ctx.state.builder.AddFunction({AtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), - old, Pointer(ctx, mem, index, pc), - ConstantU32(ctx.state, scope), - ConstantU32(ctx.state, MemorySemanticsNone), value}); - if (mem.kind == IR::ResourceKind::Lds) { - const auto semantics = - MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; - ctx.state.builder.AddFunction({OpMemoryBarrier, - ConstantU32(ctx.state, ScopeWorkgroup), - ConstantU32(ctx.state, semantics)}); - } else { - EmitDeviceAtomicMemoryBarrier(ctx.state); - } - return old; - }); + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), + [&]() { + const auto value = ctx.Arg(inst, inst.NumArgs() - 2); + const auto old = ctx.state.builder.AllocateId(); + const auto scope = + mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; + ctx.state.builder.AddFunction( + {AtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old, + EmitMemoryElementPointer(ctx.state, access.resource, access.index), + ConstantU32(ctx.state, scope), ConstantU32(ctx.state, MemorySemanticsNone), + value}); + if (mem.kind == IR::ResourceKind::Lds) { + const auto semantics = + MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; + ctx.state.builder.AddFunction({OpMemoryBarrier, + ConstantU32(ctx.state, ScopeWorkgroup), + ConstantU32(ctx.state, semantics)}); + } else { + EmitDeviceAtomicMemoryBarrier(ctx.state); + } + return old; + }); }); return return_value ? result : ConstantU32(ctx.state, 0); } @@ -479,41 +476,46 @@ uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t sour uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, bool max_value) { - const auto pc = inst.Flags().pc; return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto index = DwordIndex(ctx, inst, mem); - return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - const auto pointer = Pointer(ctx, mem, index, pc); - const auto source = ctx.Arg(inst, inst.NumArgs() - 2); - return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) { - return FloatAtomicReplacement(ctx.state, old, source, max_value); - }); - }); + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + const auto pointer = + EmitMemoryElementPointer(ctx.state, access.resource, access.index); + const auto source = ctx.Arg(inst, inst.NumArgs() - 2); + return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) { + return FloatAtomicReplacement(ctx.state, old, source, max_value); + }); + }); }); } uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, bool max_value) { - const auto pc = inst.Flags().pc; EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { - const auto index = DwordIndex(ctx, inst, mem); - EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() { - ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)}); - const auto data = ctx.state.builder.AllocateId(); - ctx.state.builder.AddFunction( - {OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable}); - AtomicUpdate(ctx.state, Pointer(ctx, mem, index, pc), mem.kind, [&](uint32_t old) { - const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old); - const auto compare_f = - Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3)); - const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data); - const auto compare = Binary( - ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, TypeBool(ctx.state), - max_value ? old_f : compare_f, max_value ? compare_f : old_f); - return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), - Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f)); - }); - }); + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + EmitIfCondition( + ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { + ctx.state.builder.AddFunction( + {OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)}); + const auto data = ctx.state.builder.AllocateId(); + ctx.state.builder.AddFunction( + {OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable}); + AtomicUpdate( + ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index), + mem.kind, [&](uint32_t old) { + const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old); + const auto compare_f = + Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 3)); + const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data); + const auto compare = + Binary(ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, + TypeBool(ctx.state), max_value ? old_f : compare_f, + max_value ? compare_f : old_f); + return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), + Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f)); + }); + }); }); return 0; } @@ -526,8 +528,12 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo const auto size = Binary(state, OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); const auto address = Binary(state, OpIAdd, TypeU32(state), base, ConstantU32(state, ctx.Memory(inst).offset)); - const auto index = + const auto raw_index = Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); + auto mem = ctx.Memory(inst); + mem.kind = gds ? IR::ResourceKind::Gds : IR::ResourceKind::Lds; + const auto access = PrepareMemoryResourceAccess(state, mem); + const auto index = EmitMemoryElementIndex(state, access, raw_index); const auto exec = ctx.Arg(inst, 2); const auto ballot = state.builder.AllocateId(); state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot, @@ -546,8 +552,7 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo ConstantU32(state, ScopeSubgroup), ballot}); const auto is_first = Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), first); - const auto storage_bounds = - gds ? EmitGdsElementInBounds(state, index) : EmitLdsElementInBounds(state, index); + const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); const auto m0_bounds = gds ? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) : Binary(state, OpULessThan, TypeBool(state), @@ -556,11 +561,10 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bo state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds))); const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { const auto value = state.builder.AllocateId(); - state.builder.AddFunction( - {append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value, - gds ? EmitGdsElementPointer(state, index) : EmitLdsElementPointer(state, index), - ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), - ConstantU32(state, MemorySemanticsNone), count}); + state.builder.AddFunction({append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value, + EmitMemoryElementPointer(state, access, index), + ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup), + ConstantU32(state, MemorySemanticsNone), count}); return value; }); const auto result = state.builder.AllocateId(); @@ -597,13 +601,14 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { Binary(state, OpIAdd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, mem.offset)); const auto index = Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); - const auto pc = inst.Flags().pc; - const auto condition = EmitStorageBufferElementInBounds(state, mem, index, pc); + const auto access = PrepareMemoryResourceAccess(state, mem); + const auto element = EmitMemoryElementIndex(state, access, index); + const auto condition = EmitMemoryElementInBounds(state, access, element); ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { const auto value = state.builder.AllocateId(); state.builder.AddFunction( {OpLoad, TypeU32(state), value, - EmitStorageBufferElementPointer(state, mem, index, pc)}); + EmitMemoryElementPointer(state, access, element)}); return value; })); return true; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 692753b37..aa34caca2 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -36,9 +36,7 @@ uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint3 return ret; } -uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc) { - (void)use_pc; +uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) { if (mem.resource >= state.program.info.buffers.size()) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); @@ -46,9 +44,7 @@ uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryIn return state.program.info.buffers[mem.resource].packed_stride; } -Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc) { - (void)use_pc; +Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem) { if (mem.resource >= state.program.info.buffers.size()) { ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); @@ -67,92 +63,6 @@ void EmitStorageBufferOffsets(EmitterState& state) { } } -bool IsAddressMemoryKind(IR::ResourceKind kind) { - return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat || - kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch; -} - -DescriptorResourceBinding -StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) { - (void)use_pc; - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); - if (state.storage_buffer_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource, - "storage buffer descriptor array was not emitted"); - } - return binding; -} - -uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, - uint32_t use_pc) { - if (IsAddressMemoryKind(mem.kind)) { - return index; - } - const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); - return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]); -} - -uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t use_pc) { - if (IsAddressMemoryKind(mem.kind)) { - if (state.address_memory_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - mem.resource, "address memory binding was not emitted"); - } - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), pointer, - state.address_memory_variable, - ConstantU32(state, binding.array_index)}); - return pointer; - } - const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), pointer, - state.storage_buffer_variable, - ConstantU32(state, binding.array_index)}); - return pointer; -} - -uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t index, uint32_t use_pc) { - index = EmitStorageBufferIndex(state, mem, index, use_pc); - const auto object = EmitStorageBufferObjectPointer(state, mem, use_pc); - const auto length = state.builder.AllocateId(); - const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction({OpArrayLength, TypeU32(state), length, object, 0}); - state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, length}); - return in_bounds; -} - -uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem, - uint32_t index, uint32_t use_pc) { - index = EmitStorageBufferIndex(state, mem, index, use_pc); - if (IsAddressMemoryKind(mem.kind)) { - if (state.address_memory_variable == 0) { - ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, - mem.resource, "address memory binding was not emitted"); - } - const auto binding = - ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, - state.address_memory_variable, - ConstantU32(state, binding.array_index), ConstantU32(state, 0), - index}); - return pointer; - } - const auto binding = StorageBufferBindingForMemory(state, mem, use_pc); - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, - state.storage_buffer_variable, - ConstantU32(state, binding.array_index), ConstantU32(state, 0), - index}); - return pointer; -} - uint32_t LdsDwordCount(const EmitterState& state) { return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u; } @@ -169,34 +79,89 @@ static void EnsureLdsStorage(EmitterState& state) { state.builder.AddName(state.lds_variable, "lds_dwords"); } -uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) { - EnsureLdsStorage(state); - const auto pointer = state.builder.AllocateId(); +MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) { + MemoryResourceAccess access {.kind = mem.kind}; + switch (mem.kind) { + case IR::ResourceKind::Lds: + EnsureLdsStorage(state); + access.object_pointer = state.lds_variable; + access.length = ConstantU32(state, LdsDwordCount(state)); + return access; + case IR::ResourceKind::Gds: + if (state.gds_variable == 0) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Gds, mem.resource, + "GDS binding was not emitted"); + } + access.object_pointer = state.gds_variable; + break; + case IR::ResourceKind::ScalarAddress: + case IR::ResourceKind::Flat: + case IR::ResourceKind::Global: + case IR::ResourceKind::Scratch: { + if (state.address_memory_variable == 0) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory, + mem.resource, + "address memory binding was not emitted"); + } + const auto binding = ResourceForDescriptor( + state, IR::DescriptorBindingKind::AddressMemory, mem.resource); + access.object_pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), + access.object_pointer, state.address_memory_variable, + ConstantU32(state, binding.array_index)}); + break; + } + case IR::ResourceKind::ScalarBuffer: + case IR::ResourceKind::Buffer: { + if (state.storage_buffer_variable == 0) { + ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, + mem.resource, + "storage buffer descriptor array was not emitted"); + } + const auto binding = + ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); + access.object_pointer = state.builder.AllocateId(); + state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state), + access.object_pointer, state.storage_buffer_variable, + ConstantU32(state, binding.array_index)}); + access.index_offset = state.storage_buffer_offsets[binding.array_index]; + access.add_index_offset = true; + break; + } + default: EXIT("unsupported memory resource kind: %u\n", static_cast(mem.kind)); + } + access.length = state.builder.AllocateId(); state.builder.AddFunction( - {OpAccessChain, - TypeLdsElementPointer(state, state.stage == ShaderType::Compute ? StorageClassWorkgroup - : StorageClassFunction), - pointer, state.lds_variable, index}); - return pointer; + {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0}); + return access; } -uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) { - EnsureLdsStorage(state); +uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access, + uint32_t raw_index) { + return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index; +} + +uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, + uint32_t index) { const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction( - {OpULessThan, TypeBool(state), in_bounds, index, ConstantU32(state, LdsDwordCount(state))}); + state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length}); return in_bounds; } -uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index, - uint32_t use_pc) { - if (mem.kind == IR::ResourceKind::Lds) { - return EmitLdsElementPointer(state, index); +uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access, + uint32_t index) { + const auto pointer = state.builder.AllocateId(); + if (access.kind == IR::ResourceKind::Lds) { + state.builder.AddFunction({OpAccessChain, + TypeLdsElementPointer(state, state.stage == ShaderType::Compute + ? StorageClassWorkgroup + : StorageClassFunction), + pointer, access.object_pointer, index}); + } else { + state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, + access.object_pointer, ConstantU32(state, 0), index}); } - if (mem.kind == IR::ResourceKind::Gds) { - return EmitGdsElementPointer(state, index); - } - return EmitStorageBufferElementPointer(state, mem, index, use_pc); + return pointer; } uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) { @@ -338,21 +303,6 @@ void EmitDeviceAtomicMemoryBarrier(EmitterState& state) { {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)}); } -uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) { - const auto length = state.builder.AllocateId(); - const auto in_bounds = state.builder.AllocateId(); - state.builder.AddFunction({OpArrayLength, TypeU32(state), length, state.gds_variable, 0}); - state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, length}); - return in_bounds; -} - -uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) { - const auto pointer = state.builder.AllocateId(); - state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, - state.gds_variable, ConstantU32(state, 0), index}); - return pointer; -} - uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) { if ((control & 0xc000u) == 0xc000u) { const uint32_t mask = control & 0x1fu; diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp index f4283bf5a..ee9656ee8 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp @@ -71,15 +71,16 @@ std::string_view ValueOpcodeName(ValueOpcode opcode) { } bool HasSideEffects(ValueOpcode opcode) { + const auto buffer_access = BufferAccessOf(opcode); + if (buffer_access == BufferAccess::Write || buffer_access == BufferAccess::Atomic) { + return true; + } switch (opcode) { case ValueOpcode::Reference: case ValueOpcode::ReferenceU32: case ValueOpcode::StoreAddressU8: case ValueOpcode::StoreAddressU16: case ValueOpcode::StoreAddressU32: - case ValueOpcode::StoreBufferU8: - case ValueOpcode::StoreBufferU16: - case ValueOpcode::StoreBufferU32: case ValueOpcode::WriteSharedU8: case ValueOpcode::WriteSharedU16: case ValueOpcode::WriteSharedU32: @@ -110,18 +111,6 @@ bool HasSideEffects(ValueOpcode opcode) { case ValueOpcode::GdsAtomicAnd32: case ValueOpcode::GdsAtomicOr32: case ValueOpcode::GdsAtomicXor32: - case ValueOpcode::BufferAtomicSwap32: - case ValueOpcode::BufferAtomicIAdd32: - case ValueOpcode::BufferAtomicISub32: - case ValueOpcode::BufferAtomicSMin32: - case ValueOpcode::BufferAtomicUMin32: - case ValueOpcode::BufferAtomicSMax32: - case ValueOpcode::BufferAtomicUMax32: - case ValueOpcode::BufferAtomicAnd32: - case ValueOpcode::BufferAtomicOr32: - case ValueOpcode::BufferAtomicXor32: - case ValueOpcode::BufferAtomicFMin32: - case ValueOpcode::BufferAtomicFMax32: case ValueOpcode::ImageAtomicIAdd32: case ValueOpcode::ImageAtomicUMin32: case ValueOpcode::ImageAtomicUMax32: @@ -139,4 +128,29 @@ bool HasSideEffects(ValueOpcode opcode) { } } +BufferAccess BufferAccessOf(ValueOpcode opcode) { + switch (opcode) { + case ValueOpcode::ReadConstBuffer: + case ValueOpcode::LoadBufferU8: + case ValueOpcode::LoadBufferU16: + case ValueOpcode::LoadBufferU32: return BufferAccess::Read; + case ValueOpcode::StoreBufferU8: + case ValueOpcode::StoreBufferU16: + case ValueOpcode::StoreBufferU32: return BufferAccess::Write; + case ValueOpcode::BufferAtomicSwap32: + case ValueOpcode::BufferAtomicIAdd32: + case ValueOpcode::BufferAtomicISub32: + case ValueOpcode::BufferAtomicSMin32: + case ValueOpcode::BufferAtomicUMin32: + case ValueOpcode::BufferAtomicSMax32: + case ValueOpcode::BufferAtomicUMax32: + case ValueOpcode::BufferAtomicAnd32: + case ValueOpcode::BufferAtomicOr32: + case ValueOpcode::BufferAtomicXor32: + case ValueOpcode::BufferAtomicFMin32: + case ValueOpcode::BufferAtomicFMax32: return BufferAccess::Atomic; + default: return BufferAccess::None; + } +} + } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h index c86fb7cc0..c2040fcc4 100644 --- a/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h +++ b/src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h @@ -16,6 +16,8 @@ enum class ValueOpcode { Count, }; +enum class BufferAccess { None, Read, Write, Atomic }; + struct DppMoveFlags { uint16_t control = 0; uint8_t row_mask = 0xf; @@ -52,6 +54,7 @@ static_assert(std::is_trivially_copyable_v); [[nodiscard]] Type ArgTypeOf(ValueOpcode opcode, size_t index); [[nodiscard]] size_t NumArgsOf(ValueOpcode opcode); [[nodiscard]] bool HasSideEffects(ValueOpcode opcode); +[[nodiscard]] BufferAccess BufferAccessOf(ValueOpcode opcode); [[nodiscard]] std::string_view ValueOpcodeName(ValueOpcode opcode); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index 877b98592..f137a1a43 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -25,46 +25,6 @@ const char* StageName(ShaderType stage) { } } -bool IsBufferAtomic(ValueOpcode op) { - switch (op) { - case ValueOpcode::BufferAtomicSwap32: - case ValueOpcode::BufferAtomicIAdd32: - case ValueOpcode::BufferAtomicISub32: - case ValueOpcode::BufferAtomicSMin32: - case ValueOpcode::BufferAtomicUMin32: - case ValueOpcode::BufferAtomicSMax32: - case ValueOpcode::BufferAtomicUMax32: - case ValueOpcode::BufferAtomicAnd32: - case ValueOpcode::BufferAtomicOr32: - case ValueOpcode::BufferAtomicXor32: - case ValueOpcode::BufferAtomicFMin32: - case ValueOpcode::BufferAtomicFMax32: return true; - default: return false; - } -} - -bool IsBufferStore(ValueOpcode op) { - switch (op) { - case ValueOpcode::StoreBufferU8: - case ValueOpcode::StoreBufferU16: - case ValueOpcode::StoreBufferU32: return true; - default: return IsBufferAtomic(op); - } -} - -bool IsBuffer(ValueOpcode op) { - switch (op) { - case ValueOpcode::ReadConstBuffer: - case ValueOpcode::LoadBufferU8: - case ValueOpcode::LoadBufferU16: - case ValueOpcode::LoadBufferU32: - case ValueOpcode::StoreBufferU8: - case ValueOpcode::StoreBufferU16: - case ValueOpcode::StoreBufferU32: return true; - default: return IsBufferAtomic(op); - } -} - bool IsAddressStore(ValueOpcode op) { switch (op) { case ValueOpcode::StoreAddressU8: @@ -393,7 +353,8 @@ private: for (const auto* block: m_values.blocks) { for (const auto& inst: *block) { const auto op = inst.GetOpcode(); - if ((!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) || &inst == &owner) { + if ((BufferAccessOf(op) == BufferAccess::None && !IsAddress(op) && !IsImage(op)) || + &inst == &owner) { continue; } if (inst.Flags().index == index) { @@ -669,8 +630,9 @@ private: static void Merge(BufferResource& resource, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) { - const bool atomic = IsBufferAtomic(op); - const bool write = IsBufferStore(op); + const auto access = BufferAccessOf(op); + const bool atomic = access == BufferAccess::Atomic; + const bool write = access == BufferAccess::Write || atomic; resource.first_use_pc = std::min(resource.first_use_pc, pc); resource.max_byte_extent = std::max(resource.max_byte_extent, ByteExtent(memory)); resource.read = resource.read || !write || atomic; @@ -813,7 +775,7 @@ private: bool Collect(Inst& inst, std::string* error) { const auto op = inst.GetOpcode(); - if (!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) { + if (BufferAccessOf(op) == BufferAccess::None && !IsAddress(op) && !IsImage(op)) { return true; } const auto flags = inst.Flags(); @@ -832,7 +794,7 @@ private: uint32_t source = 0; uint32_t resource = 0; - if (IsBuffer(op)) { + if (BufferAccessOf(op) != BufferAccess::None) { if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source, error)) { return false; diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 50b1f8074..9f2670924 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -12157,7 +12157,6 @@ TestCase Vop2SdwaSubNcExactByte2Destination() { test.code = code; test.expected = {0x000b0002u}; test.opcodes = {O::V_MOV_B32, O::V_SUB_NC_U32, O::BUFFER_STORE_DWORD, O::S_ENDPGM}; - test.required_spirv = {"OpISub", "OpShiftLeftLogical"}; return test; } @@ -15338,16 +15337,21 @@ TestCase BufferStoreFormatXAddTidUsesLaneIndex() { TestCase test; test.name = "BufferStoreFormatXAddTidUsesLaneIndex"; test.code = code; - test.initial = std::vector(4, 0); - test.expected = std::vector(4, 0x12345678u); + test.initial = std::vector(64, 0); + test.expected = std::vector(64, 0); + std::fill_n(test.expected.begin(), 32, 0x12345678u); test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM}; - test.compute_info.threads_num[0] = 4; + test.compute_info.threads_num[0] = 64; test.compute_info.threads_num[1] = 1; test.compute_info.threads_num[2] = 1; - test.compute_info.thread_ids_num = 1; + test.compute_info.thread_ids_num = 0; + test.compute_info.wave_size = 32; test.has_compute_info = true; - test.user_data = MakeStructuredStorageBufferData(4, 4, true); + test.user_data = MakeStructuredStorageBufferData(4, 64, true); test.has_user_data = true; + test.required_spirv = {"BuiltIn SubgroupLocalInvocationId"}; + test.forbidden_spirv = {"BuiltIn LocalInvocationId", + "OpGroupNonUniformBallot"}; return test; } @@ -17457,7 +17461,9 @@ void CheckIndirectImageKeySwitch() { ShaderComputeInputInfo compute{}; std::vector spirv; - ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); + Require(name, "SPIR-V requirements", + ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, &error), + error.c_str()); Require(name, "SPIR-V emit", ShaderRecompiler::Spirv::EmitProgram( program, snapshot, {.compute = &compute}, spirv, &error), diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 983349ff9..806934161 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -982,7 +982,10 @@ void TestSpirvRequirementsAnalysis() { block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)}); export_value.SetFlags(ExportFlags{.index = 0}); - ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); + std::string analysis_error; + Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, + &analysis_error), + "initial SPIR-V requirements analysis failed"); const auto requirements = *program.spirv_requirements; Check(requirements.requires_exact_subgroup && requirements.subgroup_ballot && requirements.subgroup_shuffle && @@ -993,7 +996,9 @@ void TestSpirvRequirementsAnalysis() { "consolidated SPIR-V requirements missed an IR dependency"); program.stage = ShaderType::Compute; - ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); + Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, + &analysis_error), + "compute SPIR-V requirements analysis failed"); const auto compute_requirements = *program.spirv_requirements; Check(!compute_requirements.function_lds && !compute_requirements.pixel_valid_mask, @@ -1001,14 +1006,69 @@ void TestSpirvRequirementsAnalysis() { program.stage = ShaderType::Pixel; shared.SetFlags(MemoryFlags{.index = 1}); + analysis_error.clear(); + Check(!ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, + &analysis_error) && + !program.spirv_requirements.has_value() && + Common::ContainsStr(analysis_error, "invalid memory metadata"), + "invalid shared-memory requirement metadata was accepted"); + + shared.SetFlags(MemoryFlags{.index = 0}); export_value.SetFlags(ExportFlags{.index = 1}); - ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); - const auto invalid_metadata = *program.spirv_requirements; - Check(!invalid_metadata.function_lds && !invalid_metadata.pixel_valid_mask, - "invalid SPIR-V requirement metadata was accepted"); + analysis_error.clear(); + Check(!ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, + &analysis_error) && + !program.spirv_requirements.has_value() && + Common::ContainsStr(analysis_error, "invalid metadata"), + "invalid export requirement metadata was accepted"); + + Program add_tid; + add_tid.stage = ShaderType::Compute; + add_tid.info.buffers.resize(1); + add_tid.info.buffers[0].packed_stride = 1u << 20u; + add_tid.values = std::make_shared(); + add_tid.values->block_storage.push_back(std::make_unique()); + auto *add_tid_block = add_tid.values->block_storage.back().get(); + add_tid.values->blocks.push_back(add_tid_block); + add_tid.values->memory_info.push_back( + {.kind = ResourceKind::Buffer, .resource = 0}); + auto &buffer = add_tid_block->AppendNewInst( + ValueOpcode::GetBufferResource, + {Value(0u), Value(0u), Value(0u), Value(0u)}); + auto &load = add_tid_block->AppendNewInst( + ValueOpcode::LoadBufferU32, + {Value(&buffer), Value(0u), Value(0u), Value(0u), Value(true)}); + load.SetFlags(MemoryFlags{.index = 0}); + Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid, + &analysis_error), + "ADD_TID SPIR-V requirements analysis failed"); + const auto add_tid_requirements = *add_tid.spirv_requirements; + Check(add_tid_requirements.requires_exact_subgroup && + add_tid_requirements.subgroup_local_invocation_id && + !add_tid_requirements.subgroup_ballot && + !add_tid_requirements.subgroup_shuffle, + "buffer ADD_TID requested the wrong subgroup contract"); + + add_tid.values->memory_info[0].resource = 1; + analysis_error.clear(); + Check(!ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid, + &analysis_error) && + !add_tid.spirv_requirements.has_value() && + Common::ContainsStr(analysis_error, "invalid resource metadata"), + "invalid buffer requirement metadata was accepted"); + add_tid.values->memory_info[0].resource = 0; + + add_tid.stage = ShaderType::Vertex; + analysis_error.clear(); + Check(!ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid, + &analysis_error) && + Common::ContainsStr(analysis_error, "only valid for compute"), + "graphics buffer ADD_TID was accepted without an exact guest lane ID"); Program empty; - ShaderRecompiler::Spirv::AnalyzeProgramRequirements(empty); + Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(empty, + &analysis_error), + "empty SPIR-V requirements analysis failed"); const auto empty_requirements = *empty.spirv_requirements; Check(!empty_requirements.requires_exact_subgroup && !empty_requirements.subgroup_ballot && @@ -1036,10 +1096,15 @@ void TestNativeSubgroupPolicy() { block->AppendNewInst(opcode, {Value(true)}); } else if (opcode == ValueOpcode::DataAppend) { auto &resource = block->AppendNewInst(ValueOpcode::GetLdsResource); - block->AppendNewInst(opcode, {Value(&resource), Value(0u), Value(true), - Value(1u), Value(0u)}); + program.values->memory_info.push_back({.kind = ResourceKind::Lds}); + auto &append = + block->AppendNewInst(opcode, {Value(&resource), Value(0u), + Value(true), Value(1u), Value(0u)}); + append.SetFlags(MemoryFlags{.index = 0}); } - ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); + std::string error; + Check(ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program, &error), + "native subgroup requirements analysis failed"); }; GraphicContext context; @@ -10112,8 +10177,8 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { EncodeSopp(0x01), }; const auto wide_result = compile("wide-buffer", wide_buffer, - {.words = 1988, - .instructions = 466, + {.words = 915, + .instructions = 254, .runtime_arrays = 1, .variables = 2, .loads = 9,