diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp index 6ce7612bc..f1839d91f 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp @@ -958,10 +958,9 @@ uint32_t IndirectBufferInBounds(EmitterState& state, const IndirectBufferAccess& state, spv::OpIAdd, TypeU32(state), buffer.offset, ConstantU32(state, displacement)); const auto has_payload = [&](uint32_t size) { return AndCondition( - state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size, - ConstantU32(state, bytes)), + state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), size, bytes), Binary(state, spv::OpULessThanEqual, TypeBool(state), offset, - Binary(state, spv::OpISub, TypeU32(state), size, ConstantU32(state, bytes)))); + Binary(state, spv::OpISub, TypeU32(state), size, bytes))); }; const auto structured = AndCondition( state, buffer.index_in_bounds, @@ -1002,7 +1001,8 @@ uint32_t LoadIndirectBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_ ConstantU64(state, component * 4u)); values[component] = EmitValueOrZeroIfCondition( state, AndCondition(state, valid_format, - IndirectBufferInBounds(state, buffer, component * 4u, 4u, false)), + IndirectBufferInBounds(state, buffer, component * 4u, + ConstantU32(state, 4u), false)), [&]() { return LoadBdaDword(ctx, address); }); } return ConstructU32Composite(state, components, values); @@ -1012,9 +1012,54 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) { auto& state = ctx.state; return EmitValueOrZeroIfCondition(state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() { const auto buffer = PrepareIndirectBuffer(ctx, inst); + constexpr auto format_count = + static_cast(Prospero::BufferFormat::k32_32_32_32Float) + 1u; + const auto geometry = EmitIndexSwitch( + state, buffer.format, format_count, TypeU32Composite(state, 4), [&](uint32_t format) { + const auto info = Format::GetFormatInfo(static_cast(format)); + return state.builder.Constant( + spv::OpConstantComposite, TypeU32Composite(state, 4), + ConstantU32(state, info.byte_size), + ConstantU32(state, std::max(1u, info.component_count)), + ConstantU32(state, info.packed_bitfield ? 0u : info.component_bits[0] / 8u), + ConstantU32(state, info.byte_size == 0u ? 0u : std::min(4u, info.byte_size) - 1u)); + }); + const auto field = [&](uint32_t index) { + const auto value = state.builder.AllocateId(); + state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, geometry, index); + return value; + }; + const auto bytes = field(0); + const auto component = Binary( + state, spv::OpUMod, TypeU32(state), + Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), field(1)); + const auto alignment = Unary(state, spv::OpUConvert, TypeU64(state), field(3)); + const auto base = Binary(state, spv::OpBitwiseAnd, TypeU64(state), buffer.address, + Unary(state, spv::OpNot, TypeU64(state), alignment)); + const auto address = Binary( + state, spv::OpIAdd, TypeU64(state), base, + Unary(state, spv::OpUConvert, TypeU64(state), + Binary(state, spv::OpIMul, TypeU32(state), component, field(2)))); + const auto memory = AndCondition( + state, Binary(state, spv::OpINotEqual, TypeBool(state), bytes, ConstantU32(state, 0)), + Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector, + ConstantU32(state, 4))); + const auto raw = EmitValueOrZeroIfCondition( + state, AndCondition(state, memory, IndirectBufferInBounds(state, buffer, 0u, bytes, true)), + [&]() { + // Format alignment and component spacing keep every selected component within one DWORD. + const auto word = LoadBdaDword( + ctx, Binary(state, spv::OpBitwiseAnd, TypeU64(state), address, + ConstantU64(state, ~uint64_t {3}))); + const auto byte = Binary( + state, spv::OpBitwiseAnd, TypeU32(state), + Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3)); + return Binary(state, spv::OpShiftRightLogical, TypeU32(state), word, + Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, + ConstantU32(state, 3))); + }); return EmitIndexSwitch( - state, buffer.format, static_cast(Prospero::BufferFormat::k32_32_32_32Float) + 1u, - TypeU32(state), [&](uint32_t format) { + state, buffer.format, format_count, TypeU32(state), [&](uint32_t format) { const auto info = Format::GetFormatInfo(static_cast(format)); if (info.type == Format::ComponentType::Unknown) return ConstantU32(state, 0); const auto constant = Select( @@ -1025,28 +1070,16 @@ uint32_t LoadIndirectFormattedX(ValueEmitContext& ctx, const IR::Inst& inst) { state, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), buffer.selector, ConstantU32(state, 4)), TypeU32(state), constant, [&]() { - const auto in_bounds = IndirectBufferInBounds(state, buffer, 0u, info.byte_size, true); - const auto base = Binary( - state, spv::OpBitwiseAnd, TypeU64(state), buffer.address, - ConstantU64(state, ~(uint64_t(std::min(4u, info.byte_size)) - 1u))); - const auto load = [&](uint32_t component, uint32_t bits) { - const auto offset = Format::GetFormatComponentByteOffset(info, component); - const auto address = offset == 0u - ? base - : Binary(state, spv::OpIAdd, TypeU64(state), - base, ConstantU64(state, offset)); - return LoadBda(ctx, address, in_bounds, bits); - }; const auto emit_component = [&](uint32_t component) { return LoadFormattedComponent( ctx, info, {FormattedSourceKind::Memory, component}, - [&](uint32_t source) { return load(source, 32u); }, load); + [&](uint32_t) { return raw; }, + [&](uint32_t, uint32_t bits) { + return Binary(state, spv::OpBitwiseAnd, TypeU32(state), raw, + ConstantU32(state, bits == 8u ? 0xffu : 0xffffu)); + }); }; - if (info.component_count == 1u) return emit_component(0u); - const auto component = Binary( - state, spv::OpUMod, TypeU32(state), - Binary(state, spv::OpISub, TypeU32(state), buffer.selector, ConstantU32(state, 4)), - ConstantU32(state, info.component_count)); + if (!info.packed_bitfield) return emit_component(0u); return EmitIndexSwitch(state, component, info.component_count, TypeU32(state), emit_component); }); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index a72a598ee..f6525a4ef 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -36,6 +36,7 @@ #include "graphics/host_gpu/renderer/sync.h" #include "graphics/host_gpu/vulkanCommon.h" #include "graphics/presentation/window/windowInternal.h" +#include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/Tessellation.h" #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" @@ -1356,6 +1357,7 @@ struct TestCase { std::vector storage_buffer_offsets; std::vector> storage_buffer_ranges; std::vector bda_mappings; + std::optional fault_free_address; bool expand_shader_data_storage = false; bool expected_force_point_sampler = false; float expected_float_tolerance = 0.0f; @@ -16094,7 +16096,30 @@ public: vk::PipelineStageFlagBits::eHost, {}, 0, nullptr, 1, &barrier, 0, nullptr); } + Buffer fault_result; + if (test.fault_free_address) { + Require(test.name, "fault check", uses_bda, "fault check requires BDA"); + fault_result = CreateHostBuffer(test.name, sizeof(u32), + vk::BufferUsageFlagBits::eTransferDst, {}); + const vk::MemoryBarrier ready{ + .srcAccessMask = vk::AccessFlagBits::eShaderWrite | vk::AccessFlagBits::eTransferWrite, + .dstAccessMask = vk::AccessFlagBits::eTransferRead}; + cmd.pipelineBarrier(vk::PipelineStageFlagBits::eComputeShader | vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eTransfer, {}, 1, &ready, 0, nullptr, 0, nullptr); + const vk::BufferCopy region{(BufferCache::PageIndex(*test.fault_free_address) >> 5u) * sizeof(u32), + 0, sizeof(u32)}; + cmd.copyBuffer(m_fault_buffer.buffer, fault_result.buffer, 1, ®ion); + const vk::MemoryBarrier readable{.srcAccessMask = vk::AccessFlagBits::eTransferWrite, + .dstAccessMask = vk::AccessFlagBits::eHostRead}; + cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost, + {}, 1, &readable, 0, nullptr, 0, nullptr); + } EndSubmitAndFree(test.name, "dispatch", cmd); + if (test.fault_free_address) { + Require(test.name, "fault check", ReadBuffer(test.name, fault_result, 1)[0] == 0u, + "suppressed formatted access touched an unmapped page"); + DestroyBuffer(&fault_result); + } for (const auto view : sampled_mip_views) { m_device.destroyImageView(view, nullptr); } @@ -19274,7 +19299,7 @@ private: m_bda_pagetable_buffer = CreateDeviceBuffer(shader_name, BufferCache::BDA_PAGETABLE_SIZE, usage); m_fault_buffer = CreateDeviceBuffer( - shader_name, BufferCache::CACHING_NUMPAGES / 8, usage); + shader_name, BufferCache::CACHING_NUMPAGES / 8, usage | vk::BufferUsageFlagBits::eTransferSrc); } Buffer CreateHostBuffer(const char *shader_name, vk::DeviceSize size, @@ -29842,7 +29867,7 @@ TestCase BufferLoadDwordGpuSelectedDescriptors() { return BufferLoadsGpuSelectedDescriptors(1u); } -TestCase BufferLoadFormatXGpuSelectedDescriptors() { +TestCase BufferLoadFormatXGpuSelectedDescriptors(u32 wave) { using O = ShaderOpcode; using F = Prospero::BufferFormat; constexpr uint64_t GuestBase = 0x0000000110000000ull; @@ -29851,8 +29876,10 @@ TestCase BufferLoadFormatXGpuSelectedDescriptors() { u32 selector, stride, records, mode, index, offset, soffset; bool swizzle; u32 expected; + bool zero_payload = false; + bool unmapped = false; }; - constexpr DescriptorCase cases[] = { + std::vector cases = { {F::k8UInt, 4, 1, 4, 3, 1, 0, 0, false, 0x7f}, {F::k8UInt, 4, 0, 4, 3, 0, 3, 0, false, 0x80}, {F::k16UInt, 4, 2, 4, 0, 1, 0, 0, false, 0x80ff}, @@ -29881,47 +29908,87 @@ TestCase BufferLoadFormatXGpuSelectedDescriptors() { {F::kInvalid, 1, 4, 1, 0, 0, 0, 0, false, 0}, {F::k32UInt, 4, 4, 0, 2, 0, 0, 0, false, 0}, {F::k32UInt, 4, 0, 4, 3, 0, 0, 8, false, 0}, + {F::k8_8UInt, 5, 0, 16, 3, 0, 2, 0, false, 0x80}, + {F::k16_16UInt, 5, 0, 16, 3, 0, 1, 0, false, 0x80ff}, + {F::k16_16_16_16UInt, 7, 0, 16, 3, 0, 0, 0, false, 0x8001}, + {F::k16_16_16_16Float, 6, 0, 16, 3, 0, 4, 0, false, 0x3f800000}, + {F::k32_32_32UInt, 7, 0, 16, 3, 0, 0, 0, false, 0x80ff7f01}, + {F::k32_32_32UInt, 6, 0, 16, 3, 0, 0, 0, false, 0x40003c00}, + {F::k11_11_10Float, 6, 0, 16, 3, 0, 0, 0, false, 0x400c0000}, + {F::k10_11_11Float, 4, 0, 16, 3, 0, 0, 0, false, 0x44040000}, + // Whole-record OOB and constant/reserved selectors must not touch this absent BDA page. + {F::k32_32_32_32UInt, 4, 4, 1, 0, 0, 0, 0, false, 0, false, true}, + {F::k32UInt, 0, 4, 1, 0, 0, 0, 0, false, 0, false, true}, + {F::k32UInt, 1, 4, 1, 0, 0, 0, 0, false, 1, false, true}, + {F::k32Float, 1, 4, 1, 0, 0, 0, 0, false, 0x3f800000, false, true}, + {F::k32UInt, 2, 4, 1, 0, 0, 0, 0, false, 0, false, true}, + {F::k32UInt, 3, 4, 1, 0, 0, 0, 0, false, 0, false, true}, + {static_cast(127), 4, 4, 1, 0, 0, 0, 0, false, 0, false, true}, }; + // Visit every geometry entry and every selector. Zero payloads have exact zero + // results in all numeric classes; selector ONE depends only on integer vs float. + for (u32 format = 0; format <= static_cast(F::k32_32_32_32Float); ++format) { + const auto info = ShaderRecompiler::Format::GetFormatInfo(static_cast(format)); + using T = ShaderRecompiler::Format::ComponentType; + for (u32 selector = 0; selector < 8u; ++selector) { + const u32 one = info.type == T::Uint || info.type == T::Sint ? 1u : 0x3f800000u; + cases.push_back({static_cast(format), selector, 16, 1, 0, 0, 0, 0, false, + selector == 1u && info.type != T::Unknown ? one : 0u, true}); + } + } TestCase test; - test.name = "BufferLoadFormatXGpuSelectedDescriptors"; - test.initial.resize(2048); + test.name = wave == 32 ? "BufferLoadFormatXGpuSelectedDescriptors32" + : "BufferLoadFormatXGpuSelectedDescriptors64"; + const auto align256 = [](u32 bytes) { return (bytes + 255u) & ~255u; }; + const auto selection_base = align256(static_cast(cases.size()) * 4u); + const auto descriptor_base = align256(selection_base + static_cast(cases.size()) * 4u); + const auto payload_base = align256(descriptor_base + static_cast(cases.size()) * 32u); + test.initial.resize((payload_base + cases.size() * 64u) / 4u); constexpr std::array payload{0x80ff7f01u, 0x8001ff00u, 0x40003c00u, 0x3fc00000u}; for (u32 i = 0; i < std::size(cases); ++i) { const auto &input = cases[i]; - const u32 data_offset = 4096 + i * 64; + const u32 data_offset = payload_base + i * 64; + const uint64_t data_address = GuestBase + (input.unmapped ? 0x100000u : data_offset); const std::array row{ - static_cast(GuestBase + data_offset), + static_cast(data_address), 1u | (input.stride << 16u) | (input.swizzle ? 1u << 31u : 0u), input.records, input.selector | (static_cast(input.format) << 12u) | (input.mode << 28u), input.index, input.offset, input.soffset, 0}; - std::copy(row.begin(), row.end(), test.initial.begin() + 128 + i * 8); - std::copy(payload.begin(), payload.end(), test.initial.begin() + data_offset / 4); + std::copy(row.begin(), row.end(), test.initial.begin() + descriptor_base / 4u + i * 8); + if (!input.zero_payload) + std::copy(payload.begin(), payload.end(), test.initial.begin() + data_offset / 4); // Read the selection on the GPU, as GTA's descriptor waterfall does. - test.initial[64 + i] = std::size(cases) - 1 - i; + test.initial[selection_base / 4u + i] = std::size(cases) - 1 - i; test.expected.push_back(cases[std::size(cases) - 1 - i].expected); } auto &code = test.code; AppendSMovLiteral(&code, 28, 0); + AppendSMovLiteral(&code, 29, static_cast(cases.size())); const auto loop = static_cast(code.size()); code.push_back(EncodeVop1(0x01, 30, 28)); code.push_back(EncodeVop2(0x1a, 30, InlineU32(2), 30)); - code.push_back(EncodeMubuf0(0x0c, 256)); + code.push_back(EncodeMubuf0(0x0c, selection_base)); code.push_back(EncodeMubuf1(2, 0, 30)); code.push_back(EncodeVop1(0x02, 20, Vgpr(2))); code.push_back(EncodeSop2(0x1e, 20, 20, InlineU32(5))); code.push_back(EncodeSmem0(0x0b, 8, 0)); - code.push_back(EncodeSmem1(512, 20)); + code.push_back(EncodeSmem1(descriptor_base, 20)); code.push_back(EncodeVop1(0x01, 21, 12)); code.push_back(EncodeVop1(0x01, 22, 13)); code.push_back(EncodeMubuf0(0x00, 0, true, true)); code.push_back(EncodeMubuf1(4, 2, 21, 14)); AppendBufferStoreDword(&code, 4, 30); code.push_back(EncodeSop2(0x00, 28, 28, InlineU32(1))); - code.push_back(EncodeSopc(0x0a, 28, InlineU32(std::size(cases)))); + code.push_back(EncodeSopc(0x0a, 28, 29)); code.push_back(EncodeSopp(0x05, loop - static_cast(code.size()) - 1u)); AppendEnd(&code); test.bda_mappings = {{GuestBase, 0}}; + test.fault_free_address = GuestBase + 0x100000u; + test.compute_info.threads_num[0] = 1; + test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1; + test.compute_info.wave_size = wave; + test.has_compute_info = true; test.required_spirv = {"PhysicalStorageBuffer", "OpSwitch", "OpLoopMerge"}; test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::V_LSHLREV_B32, O::BUFFER_LOAD_DWORD, O::V_READFIRSTLANE_B32, O::S_LSHL_B32, @@ -29931,6 +29998,14 @@ TestCase BufferLoadFormatXGpuSelectedDescriptors() { return test; } +TestCase BufferLoadFormatXGpuSelectedDescriptors() { + return BufferLoadFormatXGpuSelectedDescriptors(32u); +} + +TestCase BufferLoadFormatXGpuSelectedDescriptors64() { + return BufferLoadFormatXGpuSelectedDescriptors(64u); +} + TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() { using O = ShaderOpcode; @@ -35790,18 +35865,24 @@ void CheckBoundedFormattedBufferCopy(VulkanHarness &vulkan) { if (count == 8u) { std::set nonuniform; std::vector stores; + u32 bda_calls = 0; for (size_t offset = 5; offset < compiled.spirv.size();) { const auto words = std::span(compiled.spirv).subspan(offset, compiled.spirv[offset] >> 16u); const auto opcode = static_cast(words[0] & 0xffffu); if (opcode == spv::OpDecorate && words[2] == spv::DecorationNonUniform) nonuniform.insert(words[1]); if (opcode == spv::OpStore) stores.push_back(words[1]); + if (opcode == spv::OpFunctionCall) ++bda_calls; offset += words.size(); } Require(test.name, "one shared indirect store per virtual half", std::ranges::count_if(stores, [&](u32 pointer) { return nonuniform.contains(pointer); }) == (wave > vulkan.SubgroupSize() ? 2 : 1), "formatted identity lowering duplicated the store per descriptor"); + Require(test.name, "shared formatted payload read", bda_calls <= 12u, + "formatted source load duplicated BDA reads per format or component"); + std::printf("[compute] BoundedFormattedBufferCopy%u: %zu SPIR-V words, %u BDA calls\n", + wave, compiled.spirv.size(), bda_calls); } auto output = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size(), true); vulkan.Dispatch(test, compiled, output); @@ -38109,6 +38190,7 @@ std::vector MakeCases() { AddCase(BufferLoadDwordGpuSelectedDescriptors); AddCase(BufferLoadDwordx3GpuSelectedDescriptors); AddCase(BufferLoadFormatXGpuSelectedDescriptors); + AddCase(BufferLoadFormatXGpuSelectedDescriptors64); AddCase(BufferStoreDwordx4DropsOnlyOutOfBoundsTail); AddCase(BufferLoadFormatXyzwRejectsPartialRecord); AddCase(BufferStoreFormatXyzwDropsPartialRecord); @@ -43968,6 +44050,7 @@ int main(int argc, char **argv) { RunCase(&vulkan, BufferLoadDwordGpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadDwordx3GpuSelectedDescriptors()); RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors()); + RunCase(&vulkan, BufferLoadFormatXGpuSelectedDescriptors64()); RunCase(&vulkan, BufferLoadFormatXRejectsPartialRecord()); RunCase(&vulkan, BufferLoadFormatXyRejectsPartialRecord()); RunCase(&vulkan, BufferLoadDwordx4SnapshotsOverlappingAddress());