diff --git a/src/graphics/shader/recompiler/Tessellation.cpp b/src/graphics/shader/recompiler/Tessellation.cpp index ee42a2345..4b194609f 100644 --- a/src/graphics/shader/recompiler/Tessellation.cpp +++ b/src/graphics/shader/recompiler/Tessellation.cpp @@ -34,8 +34,8 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local, using Registers = std::array; const auto control_points = local ? info.input_control_points : info.output_control_points; - Registers registers {}; - registers[local ? 3u : 1u] = + Registers entry {}, registers; + entry[local ? 3u : 1u] = local ? Address {Kind::Affine, 1u, 0u} : Address {Kind::PackedControlPoint}; const auto constant = [](uint32_t value) { return Address {Kind::Affine, 0u, value}; }; const auto read = [&](const Operand& operand) -> Address { @@ -85,101 +85,115 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local, ? value : Address {}; }; - // A forward branch joins its target with the fallthrough path. Definitions that - // differ between the paths are unknown there. - std::unordered_map joins; + // A branch joins its target with the fallthrough path. Definitions that differ between + // the paths are unknown there. Loop headers repeat the walk until back edges are stable. + std::unordered_map joins, loops; const auto merge = [](Registers& into, const Registers& from) { + bool changed = false; for (uint32_t index = 0; index < into.size(); index++) { - if (into[index] != from[index]) into[index] = {}; + if (into[index] == from[index]) continue; + changed |= into[index] != Address {}; + into[index] = {}; } + return changed; }; - uint32_t stride = 0, patch_begin = UINT32_MAX, patch_end = 0; - for (const auto& inst: program.instructions) { - if (const auto join = joins.find(inst.pc); join != joins.end()) { - merge(registers, join->second); - joins.erase(join); - } - if (IsDirectBranch(inst.opcode)) { - EXIT_NOT_IMPLEMENTED(inst.branch_target <= inst.pc); - if (const auto [join, inserted] = joins.try_emplace(inst.branch_target, registers); - !inserted) - merge(join->second, registers); - } - const bool local_store = IsDsWriteOpcode(inst.opcode); - const bool buffer_store = inst.opcode == Opcode::BUFFER_STORE_DWORD || - inst.opcode == Opcode::BUFFER_STORE_DWORDX2 || - inst.opcode == Opcode::BUFFER_STORE_DWORDX3 || - inst.opcode == Opcode::BUFFER_STORE_DWORDX4; - const bool control_store = - !local && buffer_store && inst.src2.kind == OperandKind::Sgpr && inst.src2.reg == 2u; - const bool control_read = - !local && (inst.opcode == Opcode::DS_READ_B32 || inst.opcode == Opcode::DS_READ2_B32); - if ((local && local_store) || control_store || control_read) { - const auto address = read(inst.src0); - if (address.kind != Kind::Affine) { - EXIT("%s tessellation address is not affine at pc 0x%08x\n", local ? "LS" : "HS", - inst.pc); + uint32_t stride = 0, patch_begin = 0, patch_end = 0; + for (bool repeat = true; repeat;) { + repeat = false; + registers = entry; + stride = 0, patch_begin = UINT32_MAX, patch_end = 0; + for (const auto& inst: program.instructions) { + if (const auto join = joins.find(inst.pc); join != joins.end()) { + merge(registers, join->second); + joins.erase(join); } - if (address.coefficient != 0u) { - const auto expected = control_read ? info.ls_stride : stride; - EXIT_NOT_IMPLEMENTED((address.coefficient & 3u) != 0u || - (expected != 0u && expected != address.coefficient)); - if (!control_read) stride = address.coefficient; - } else if (control_store) { - // Control-point-independent HS outputs are patch constants. - const auto begin = address.constant + inst.offset; - patch_begin = std::min(patch_begin, begin); - patch_end = std::max(patch_end, begin + 4u * inst.data_dwords); + if (const auto loop = loops.find(inst.pc); loop != loops.end()) + merge(registers, loop->second); + if (IsDirectBranch(inst.opcode)) { + const bool back = inst.branch_target <= inst.pc; + auto [join, inserted] = + (back ? loops : joins).try_emplace(inst.branch_target, registers); + const bool changed = !inserted && merge(join->second, registers); + repeat |= back && (inserted || changed); } - } - // Store vdata is a source, despite occupying the decoder's dst field. - if (local_store || buffer_store || inst.dst.kind != OperandKind::Vgpr) continue; - const auto lhs = read(inst.src0); - const auto rhs = read(inst.src1); - const auto third = read(inst.src2); - Address value; - switch (inst.opcode) { - case Opcode::V_BFE_U32: - if (lhs.kind == Kind::PackedControlPoint && rhs.kind == Kind::Affine && - rhs.coefficient == 0u && third.kind == Kind::Affine && - third.coefficient == 0u) { - if (rhs.constant == 8u && third.constant == 5u) value = {Kind::Affine, 1u, 0u}; - if (rhs.constant == 0u && third.constant == 8u) value = constant(0u); + const bool local_store = IsDsWriteOpcode(inst.opcode); + const bool buffer_store = inst.opcode == Opcode::BUFFER_STORE_DWORD || + inst.opcode == Opcode::BUFFER_STORE_DWORDX2 || + inst.opcode == Opcode::BUFFER_STORE_DWORDX3 || + inst.opcode == Opcode::BUFFER_STORE_DWORDX4; + const bool control_store = !local && buffer_store && + inst.src2.kind == OperandKind::Sgpr && inst.src2.reg == 2u; + if ((local && local_store) || control_store) { + const auto address = read(inst.src0); + if (address.kind != Kind::Affine) { + EXIT("%s tessellation address is not affine at pc 0x%08x\n", + local ? "LS" : "HS", inst.pc); } - break; - case Opcode::V_AND_B32: - if ((lhs.kind == Kind::PackedControlPoint && rhs == constant(0xffu)) || - (rhs.kind == Kind::PackedControlPoint && lhs == constant(0xffu))) - value = constant(0u); - break; - case Opcode::V_ADD_NC_U32: value = add(lhs, rhs); break; - case Opcode::V_MUL_LO_U32: value = multiply(lhs, rhs); break; - case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break; - case Opcode::V_MAD_U32_U24: value = add(multiply(low24(lhs), low24(rhs)), third); break; - case Opcode::V_LSHL_ADD_U32: - if (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant < 32u) - value = add(multiply(lhs, constant(1u << rhs.constant)), third); - break; - case Opcode::V_LSHLREV_B32: - if (lhs.kind == Kind::Affine && lhs.coefficient == 0u && lhs.constant < 32u) - value = multiply(rhs, constant(1u << lhs.constant)); - break; - case Opcode::V_SUB_NC_U32: - if (lhs.kind == Kind::Affine && rhs.kind == Kind::Affine && - lhs.coefficient >= rhs.coefficient && lhs.constant >= rhs.constant) - value = affine(lhs.coefficient - rhs.coefficient, lhs.constant - rhs.constant); - break; - default: break; + if (address.coefficient != 0u) { + EXIT_NOT_IMPLEMENTED((address.coefficient & 3u) != 0u || + (stride != 0u && stride != address.coefficient)); + stride = address.coefficient; + } else if (control_store) { + // Control-point-independent HS outputs are patch constants. + const auto begin = address.constant + inst.offset; + patch_begin = std::min(patch_begin, begin); + patch_end = std::max(patch_end, begin + 4u * inst.data_dwords); + } + } + // Store vdata is a source, despite occupying the decoder's dst field. + if (local_store || buffer_store || inst.dst.kind != OperandKind::Vgpr) continue; + const auto lhs = read(inst.src0); + const auto rhs = read(inst.src1); + const auto third = read(inst.src2); + Address value; + switch (inst.opcode) { + case Opcode::V_BFE_U32: + if (lhs.kind == Kind::PackedControlPoint && rhs.kind == Kind::Affine && + rhs.coefficient == 0u && third.kind == Kind::Affine && + third.coefficient == 0u) { + if (rhs.constant == 8u && third.constant == 5u) + value = {Kind::Affine, 1u, 0u}; + if (rhs.constant == 0u && third.constant == 8u) value = constant(0u); + } + break; + case Opcode::V_AND_B32: + if ((lhs.kind == Kind::PackedControlPoint && rhs == constant(0xffu)) || + (rhs.kind == Kind::PackedControlPoint && lhs == constant(0xffu))) + value = constant(0u); + break; + case Opcode::V_ADD_NC_U32: value = add(lhs, rhs); break; + case Opcode::V_MUL_LO_U32: value = multiply(lhs, rhs); break; + case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break; + case Opcode::V_MAD_U32_U24: + value = add(multiply(low24(lhs), low24(rhs)), third); + break; + case Opcode::V_LSHL_ADD_U32: + if (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant < 32u) + value = add(multiply(lhs, constant(1u << rhs.constant)), third); + break; + case Opcode::V_LSHLREV_B32: + if (lhs.kind == Kind::Affine && lhs.coefficient == 0u && lhs.constant < 32u) + value = multiply(rhs, constant(1u << lhs.constant)); + break; + case Opcode::V_SUB_NC_U32: + if (lhs.kind == Kind::Affine && rhs.kind == Kind::Affine && + lhs.coefficient >= rhs.coefficient && lhs.constant >= rhs.constant) + value = + affine(lhs.coefficient - rhs.coefficient, lhs.constant - rhs.constant); + break; + default: break; + } + if (inst.dst.sdwa_sel != 6u || inst.dst.op_sel || inst.dst.omod || inst.dst.clamp || + inst.dst.dpp) + value = {}; + for (uint32_t index = 0; + index < std::max(inst.data_dwords, 1u) && inst.dst.reg + index < registers.size(); + index++) { + registers[inst.dst.reg + index] = {}; + } + registers.at(inst.dst.reg) = value; } - if (inst.dst.sdwa_sel != 6u || inst.dst.op_sel || inst.dst.omod || inst.dst.clamp || - inst.dst.dpp) - value = {}; - for (uint32_t index = 0; - index < std::max(inst.data_dwords, 1u) && inst.dst.reg + index < registers.size(); index++) { - registers[inst.dst.reg + index] = {}; - } - registers.at(inst.dst.reg) = value; } EXIT_NOT_IMPLEMENTED(!joins.empty() || stride == 0u); (local ? info.ls_stride : info.hs_stride) = stride; @@ -318,20 +332,22 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options kind = TessellationAttribute::Factor; factors += components; } else { - kind = write ? TessellationAttribute::ControlOutput - : TessellationAttribute::EvaluationInput; - EXIT_NOT_IMPLEMENTED(write - ? options.stage != ShaderType::TessellationControl - : options.stage != ShaderType::TessellationEvaluation); + // After its barrier, the HS may read back outputs of other control points. + const bool output = options.stage == ShaderType::TessellationControl; + EXIT_NOT_IMPLEMENTED( + !output && (write || options.stage != ShaderType::TessellationEvaluation)); + kind = output ? TessellationAttribute::ControlOutput + : TessellationAttribute::EvaluationInput; if (address.IsImmediate() && address.U32() >= tess.hs_stride * tess.output_control_points) { const auto begin = address.U32() + memory.offset; EXIT_NOT_IMPLEMENTED(begin < tess.patch_base || begin + 4u * components > tess.patch_base + tess.patch_size); - kind = write ? TessellationAttribute::PatchOutput - : TessellationAttribute::PatchInput; - patches += components; + kind = output ? TessellationAttribute::PatchOutput + : TessellationAttribute::PatchInput; + // HS read-backs must not hide misplaced patch stores from the check below. + if (write || !output) patches += components; } } } else { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp index 96cbba35a..cd1d5320c 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp @@ -56,8 +56,10 @@ uint32_t TessellationPointer(ValueEmitContext& ctx, const IR::Inst& inst) { state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, attribute, component); } else { + // An HS writes only its own control point but may read any other. const auto vertex = - kind == Attribute::ControlOutput + kind == Attribute::ControlOutput && + inst.GetOpcode() == IR::ValueOpcode::SetTessellationAttribute ? EmitLaneId(state) : EmitBinaryU32(state, spv::OpUDiv, address, ConstantU32(state, stride)); state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, vertex, attribute, @@ -126,7 +128,9 @@ void DefineTessellationInterfaces(EmitterState& state) { void DefineTessellationExecutionModes(EmitterState& state) { const auto& tess = state.input_info.vertex->tess; - EXIT_NOT_IMPLEMENTED(tess.domain != 1u || tess.partitioning != 2u || + // CxVgtTfParam: triangles, integer or fractional-odd partitioning, clockwise output. + EXIT_NOT_IMPLEMENTED(tess.domain != 1u || + (tess.partitioning != 0u && tess.partitioning != 2u) || tess.output_topology != 2u); state.builder.RequireCapability(spv::CapabilityTessellation); if (state.program.stage == ShaderType::TessellationControl) { @@ -134,7 +138,9 @@ void DefineTessellationExecutionModes(EmitterState& state) { tess.output_control_points); } else { state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeTriangles); - state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSpacingFractionalOdd); + state.builder.AddExecutionMode( + state.main_func, tess.partitioning == 0u ? spv::ExecutionModeSpacingEqual + : spv::ExecutionModeSpacingFractionalOdd); state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeVertexOrderCw); } } diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 3b4cbe7cd..1dc664b82 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -43885,7 +43885,8 @@ void CheckTessellationProgram(const char *name, u32 ls_stride, u32 hs_stride) { // PPSA25468: CP0 writes patch constants below the 32 KiB offchip ring end // through mul24 addresses that reuse data VGPRs under nested EXEC masks, and -// the TES reads them back. +// the TES reads them back. As in God of War Ragnarok, the HS reads its next LS +// record, loops on a v_cmpx header and CP0 reads back another CP's outputs. void CheckTessellationPatchConstants() { using namespace ShaderRecompiler; using Attribute = IR::TessellationAttribute; @@ -43899,7 +43900,7 @@ void CheckTessellationPatchConstants() { local.push_back(EncodeDs1(0, 2, 20)); local.push_back(EncodeSop1(0x20, 0, 6)); - const auto make_control = [&](bool divergent) { + const auto make_control = [&](bool divergent, bool carried) { std::vector control; AppendVop3(&control, 0x365, 0, 193u, InlineU32(0)); AppendVop3(&control, 0x366, 0, 193u, Vgpr(0)); @@ -43912,7 +43913,10 @@ void CheckTessellationPatchConstants() { control.push_back(0xffu); control.push_back(EncodeVop2(0x0b, 0, 255u, 16)); // v_mul_u32_u24 v0, hs_stride, v16 control.push_back(hs_stride); - AppendVop3(&control, 0x169, 1, Vgpr(16), 255u); // v_mul_lo_u32 v1, v16, ls_stride + control.push_back(EncodeVopc(0xc4, InlineU32(2), 16)); // v_cmp_gt_u32 vcc, 2, v16 + control.push_back(EncodeVop2(0x25, 1, InlineU32(1), 16)); // v_add_nc_u32 v1, 1, v16 + control.push_back(EncodeVop2(0x01, 1, InlineU32(0), 1)); // v_cndmask_b32 v1, 0, v1, vcc + AppendVop3(&control, 0x169, 1, Vgpr(1), 255u); // v_mul_lo_u32 v1, v1, ls_stride control.push_back(ls_stride); AppendVop3(&control, 0x143, 17, 255u, Vgpr(50), Vgpr(0)); control.push_back(3 * hs_stride); @@ -43932,12 +43936,33 @@ void CheckTessellationPatchConstants() { control.push_back(EncodeVop2(0x25, 17, InlineU32(4), 17)); // v_add_nc_u32 v17, 4, v17 } control[skip_scale] = EncodeSopp(0x06, control.size() - skip_scale - 1u); + control.push_back(EncodeSop1(0x04, 6, 126)); // s_mov_b64 s[6:7], exec + control.push_back(EncodeSMovB32(5, InlineU32(0))); // s_mov_b32 s5, 0 + const auto loop = control.size(); + control.push_back(EncodeVopc(0xd1, 5, 16)); // v_cmpx_lt_u32 s5, v16 + const auto exit_loop = control.size(); + control.push_back(0u); // s_cbranch_execz control.push_back(EncodeMubuf0(0x1c, 24)); control.push_back(EncodeMubuf1(12, 2, 17, 2)); + // Only the back edge shows a loop-carried v17 at the store above. + if (carried) { + control.push_back(EncodeVop2(0x25, 17, InlineU32(4), 17)); // v_add_nc_u32 v17, 4, v17 + } + control.push_back(EncodeVop2(0x25, 13, InlineU32(1), 13)); // v_add_nc_u32 v13, 1, v13 + control.push_back(EncodeSop2(0x00, 5, 5, InlineU32(1))); // s_add_u32 s5, s5, 1 + control.push_back(EncodeSopp(0x02, loop - control.size() - 1u)); // s_branch + control[exit_loop] = EncodeSopp(0x08, control.size() - exit_loop - 1u); + control.push_back(EncodeSop1(0x04, 126, 6)); // s_mov_b64 exec, s[6:7] + control.push_back(EncodeSopp(0x0a)); // s_barrier // CP0 reuses data VGPR v10 for the ring address 0x8000 - (patch + 1) * 0xf0. control.push_back(EncodeVopc(0xd4, InlineU32(1), 16)); // v_cmpx_gt_u32 1, v16 const auto skip_patch = control.size(); control.push_back(0u); // s_cbranch_execz + control.push_back(EncodeVop2(0x0b, 11, 255u, 50)); // v_mul_u32_u24 v11, 3 * hs_stride, v50 + control.push_back(3 * hs_stride); + // buffer_load_dwordx2 v[11:12], v11, s[8:11], s2 offen offset:hs_stride + 16 (CP1) + control.push_back(EncodeMubuf0(0x0d, hs_stride + 16)); + control.push_back(EncodeMubuf1(11, 2, 11, 2)); control.push_back(EncodeVop2(0x25, 10, InlineU32(1), 50)); // v_add_nc_u32 v10, 1, v50 control.push_back(EncodeVop2(0x0b, 10, 255u, 10)); // v_mul_u32_u24 v10, 0xf0, v10 control.push_back(patch_stride); @@ -43959,7 +43984,7 @@ void CheckTessellationPatchConstants() { AppendEnd(&control); return control; }; - const auto control = make_control(false); + const auto control = make_control(false, false); std::vector evaluation; AppendVop3(&evaluation, 0x169, 24, 255u, Vgpr(7)); @@ -43982,13 +44007,14 @@ void CheckTessellationPatchConstants() { ShaderTessellationInputInfo tess{.input_control_points = 3, .output_control_points = 3, .domain = 1, - .partitioning = 2, + .partitioning = 0, .output_topology = 2}; AnalyzeTessellationPrograms(local, control, tess); Require(name, "reflected interface", tess.ls_stride == ls_stride && tess.hs_stride == hs_stride && tess.patch_base == 0x7f10u && tess.patch_size == 28u, - "b128 LS records, joined HS paths and CP0 patch stores must reflect the layout"); + "b128 LS records, selected LS reads, joined and looping HS paths and CP0 patch " + "stores must reflect the layout"); auto ls = CompileTessellationStage(name, local, ShaderType::Local, tess); auto hs = CompileTessellationStage(name, control, ShaderType::TessellationControl, tess); @@ -43999,6 +44025,10 @@ void CheckTessellationPatchConstants() { Require(name, "predicated patch outputs", hs.addresses[Attribute::PatchOutput] == std::set{0x7f10, 0x7f20, 0x7f24, 0x7f28}, "CP0 mul24 addresses under nested EXEC masks must reach patch constants"); + Require(name, "control point read-back", + hs.addresses[Attribute::ControlOutput] == std::set{hs_stride + 16, hs_stride + 20} && + hs.assembly.find("OpUDiv %uint %uint_96 %uint_80") != std::string::npos, + "CP0 must read CP1's outputs by address instead of its own invocation's"); Require(name, "patch inputs", ds.addresses[Attribute::PatchInput] == std::set{0x7f20, 0x7f24} && ds.addresses[Attribute::EvaluationInput] == @@ -44008,12 +44038,16 @@ void CheckTessellationPatchConstants() { HasPatchInterface(hs.assembly, "Output", 2, 5) && HasPatchInterface(ds.assembly, "Input", 2, 5), "HS and TES patch variables must share their Location and type"); + Require(name, "integer partitioning", ds.assembly.find("SpacingEqual") != std::string::npos, + "integer partitioning must map to equal spacing"); std::printf("[host] %-32s ok\n", name); #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - ExpectFatal("TessellationDivergentJoin", [&] { - ShaderTessellationInputInfo info{.input_control_points = 3, .output_control_points = 3}; - AnalyzeTessellationPrograms(local, make_control(true), info); - }); + for (const bool carried : {false, true}) { + ExpectFatal(carried ? "TessellationLoopCarriedAddress" : "TessellationDivergentJoin", [&] { + ShaderTessellationInputInfo info{.input_control_points = 3, .output_control_points = 3}; + AnalyzeTessellationPrograms(local, make_control(!carried, carried), info); + }); + } #endif }