shader: lower HS neighbour LDS reads, control-point output read-back, looped HS reflection and integer partitioning

Co-authored-by: NeySlim <4509984+NeySlim@users.noreply.github.com>

src/graphics/shader/recompiler/Tessellation.cpp

@@ -34,8 +34,8 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 	using Registers = std::array<Address, IR::NumVectorRegs>;
 
 	const auto control_points = local ? info.input_control_points : info.output_control_points;
-	Registers  registers {};
+	Registers  entry {}, registers;
-	registers[local ? 3u : 1u] =
+	entry[local ? 3u : 1u] =
 	    local ? Address {Kind::Affine, 1u, 0u} : Address {Kind::PackedControlPoint};
 	const auto constant = [](uint32_t value) { return Address {Kind::Affine, 0u, value}; };
 	const auto read     = [&](const Operand& operand) -> Address {
@@ -85,47 +85,55 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 		           ? value
 		           : Address {};
 	};
-	// A forward branch joins its target with the fallthrough path. Definitions that
+	// A branch joins its target with the fallthrough path. Definitions that differ between
-	// differ between the paths are unknown there.
+	// the paths are unknown there. Loop headers repeat the walk until back edges are stable.
-	std::unordered_map<uint32_t, Registers> joins;
+	std::unordered_map<uint32_t, Registers> joins, loops;
 	const auto merge = [](Registers& into, const Registers& from) {
+		bool changed = false;
 		for (uint32_t index = 0; index < into.size(); index++) {
-			if (into[index] != from[index]) into[index] = {};
+			if (into[index] == from[index]) continue;
+			changed |= into[index] != Address {};
+			into[index] = {};
 		}
+		return changed;
 	};
 
-	uint32_t stride = 0, patch_begin = UINT32_MAX, patch_end = 0;
+	uint32_t stride = 0, patch_begin = 0, patch_end = 0;
+	for (bool repeat = true; repeat;) {
+		repeat    = false;
+		registers = entry;
+		stride = 0, patch_begin = UINT32_MAX, patch_end = 0;
 		for (const auto& inst: program.instructions) {
 			if (const auto join = joins.find(inst.pc); join != joins.end()) {
 				merge(registers, join->second);
 				joins.erase(join);
 			}
+			if (const auto loop = loops.find(inst.pc); loop != loops.end())
+				merge(registers, loop->second);
 			if (IsDirectBranch(inst.opcode)) {
-			EXIT_NOT_IMPLEMENTED(inst.branch_target <= inst.pc);
+				const bool back = inst.branch_target <= inst.pc;
-			if (const auto [join, inserted] = joins.try_emplace(inst.branch_target, registers);
+				auto [join, inserted] =
-			    !inserted)
+				    (back ? loops : joins).try_emplace(inst.branch_target, registers);
-				merge(join->second, registers);
+				const bool changed = !inserted && merge(join->second, registers);
+				repeat |= back && (inserted || changed);
 			}
 			const bool local_store  = IsDsWriteOpcode(inst.opcode);
 			const bool buffer_store = inst.opcode == Opcode::BUFFER_STORE_DWORD ||
 			                          inst.opcode == Opcode::BUFFER_STORE_DWORDX2 ||
 			                          inst.opcode == Opcode::BUFFER_STORE_DWORDX3 ||
 			                          inst.opcode == Opcode::BUFFER_STORE_DWORDX4;
-		const bool control_store =
+			const bool control_store = !local && buffer_store &&
-		    !local && buffer_store && inst.src2.kind == OperandKind::Sgpr && inst.src2.reg == 2u;
+			                           inst.src2.kind == OperandKind::Sgpr && inst.src2.reg == 2u;
-		const bool control_read =
+			if ((local && local_store) || control_store) {
-		    !local && (inst.opcode == Opcode::DS_READ_B32 || inst.opcode == Opcode::DS_READ2_B32);
-		if ((local && local_store) || control_store || control_read) {
 				const auto address = read(inst.src0);
 				if (address.kind != Kind::Affine) {
-				EXIT("%s tessellation address is not affine at pc 0x%08x\n", local ? "LS" : "HS",
+					EXIT("%s tessellation address is not affine at pc 0x%08x\n",
-				     inst.pc);
+					     local ? "LS" : "HS", inst.pc);
 				}
 				if (address.coefficient != 0u) {
-				const auto expected = control_read ? info.ls_stride : stride;
 					EXIT_NOT_IMPLEMENTED((address.coefficient & 3u) != 0u ||
-				                     (expected != 0u && expected != address.coefficient));
+					                     (stride != 0u && stride != address.coefficient));
-				if (!control_read) stride = address.coefficient;
+					stride = address.coefficient;
 				} else if (control_store) {
 					// Control-point-independent HS outputs are patch constants.
 					const auto begin = address.constant + inst.offset;
@@ -144,7 +152,8 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 					if (lhs.kind == Kind::PackedControlPoint && rhs.kind == Kind::Affine &&
 					    rhs.coefficient == 0u && third.kind == Kind::Affine &&
 					    third.coefficient == 0u) {
-					if (rhs.constant == 8u && third.constant == 5u) value = {Kind::Affine, 1u, 0u};
+						if (rhs.constant == 8u && third.constant == 5u)
+							value = {Kind::Affine, 1u, 0u};
 						if (rhs.constant == 0u && third.constant == 8u) value = constant(0u);
 					}
 					break;
@@ -156,7 +165,9 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 				case Opcode::V_ADD_NC_U32: value = add(lhs, rhs); break;
 				case Opcode::V_MUL_LO_U32: value = multiply(lhs, rhs); break;
 				case Opcode::V_MUL_U32_U24: value = multiply(low24(lhs), low24(rhs)); break;
-			case Opcode::V_MAD_U32_U24: value = add(multiply(low24(lhs), low24(rhs)), third); break;
+				case Opcode::V_MAD_U32_U24:
+					value = add(multiply(low24(lhs), low24(rhs)), third);
+					break;
 				case Opcode::V_LSHL_ADD_U32:
 					if (rhs.kind == Kind::Affine && rhs.coefficient == 0u && rhs.constant < 32u)
 						value = add(multiply(lhs, constant(1u << rhs.constant)), third);
@@ -168,7 +179,8 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 				case Opcode::V_SUB_NC_U32:
 					if (lhs.kind == Kind::Affine && rhs.kind == Kind::Affine &&
 					    lhs.coefficient >= rhs.coefficient && lhs.constant >= rhs.constant)
-					value = affine(lhs.coefficient - rhs.coefficient, lhs.constant - rhs.constant);
+						value =
+						    affine(lhs.coefficient - rhs.coefficient, lhs.constant - rhs.constant);
 					break;
 				default: break;
 			}
@@ -176,11 +188,13 @@ void ReflectTessellationLayout(const Decoder::Program& program, bool local,
 			    inst.dst.dpp)
 				value = {};
 			for (uint32_t index = 0;
-		     index < std::max(inst.data_dwords, 1u) && inst.dst.reg + index < registers.size(); index++) {
+			     index < std::max(inst.data_dwords, 1u) && inst.dst.reg + index < registers.size();
+			     index++) {
 				registers[inst.dst.reg + index] = {};
 			}
 			registers.at(inst.dst.reg) = value;
 		}
+	}
 	EXIT_NOT_IMPLEMENTED(!joins.empty() || stride == 0u);
 	(local ? info.ls_stride : info.hs_stride) = stride;
 	if (patch_end != 0u) {
@@ -318,20 +332,22 @@ void LowerTessellationMemory(IR::Program& program, const CompileOptions& options
 					kind = TessellationAttribute::Factor;
 					factors += components;
 				} else {
-					kind = write ? TessellationAttribute::ControlOutput
+					// After its barrier, the HS may read back outputs of other control points.
+					const bool output = options.stage == ShaderType::TessellationControl;
+					EXIT_NOT_IMPLEMENTED(
+					    !output && (write || options.stage != ShaderType::TessellationEvaluation));
+					kind = output ? TessellationAttribute::ControlOutput
 					              : TessellationAttribute::EvaluationInput;
-					EXIT_NOT_IMPLEMENTED(write
-					                         ? options.stage != ShaderType::TessellationControl
-					                         : options.stage != ShaderType::TessellationEvaluation);
 					if (address.IsImmediate() &&
 					    address.U32() >= tess.hs_stride * tess.output_control_points) {
 						const auto begin = address.U32() + memory.offset;
 						EXIT_NOT_IMPLEMENTED(begin < tess.patch_base ||
 						                     begin + 4u * components >
 						                         tess.patch_base + tess.patch_size);
-						kind = write ? TessellationAttribute::PatchOutput
+						kind = output ? TessellationAttribute::PatchOutput
 						              : TessellationAttribute::PatchInput;
-						patches += components;
+						// HS read-backs must not hide misplaced patch stores from the check below.
+						if (write || !output) patches += components;
 					}
 				}
 			} else {

src/graphics/shader/recompiler/backend/spirv/spirvEmitterTessellation.cpp

@@ -56,8 +56,10 @@ uint32_t TessellationPointer(ValueEmitContext& ctx, const IR::Inst& inst) {
 		state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, attribute,
 		                          component);
 	} else {
+		// An HS writes only its own control point but may read any other.
 		const auto vertex =
-		    kind == Attribute::ControlOutput
+		    kind == Attribute::ControlOutput &&
+		            inst.GetOpcode() == IR::ValueOpcode::SetTessellationAttribute
 		        ? EmitLaneId(state)
 		        : EmitBinaryU32(state, spv::OpUDiv, address, ConstantU32(state, stride));
 		state.builder.AddFunction(spv::OpAccessChain, type, pointer, variable, vertex, attribute,
@@ -126,7 +128,9 @@ void DefineTessellationInterfaces(EmitterState& state) {
 
 void DefineTessellationExecutionModes(EmitterState& state) {
 	const auto& tess = state.input_info.vertex->tess;
-	EXIT_NOT_IMPLEMENTED(tess.domain != 1u || tess.partitioning != 2u ||
+	// CxVgtTfParam: triangles, integer or fractional-odd partitioning, clockwise output.
+	EXIT_NOT_IMPLEMENTED(tess.domain != 1u ||
+	                     (tess.partitioning != 0u && tess.partitioning != 2u) ||
 	                     tess.output_topology != 2u);
 	state.builder.RequireCapability(spv::CapabilityTessellation);
 	if (state.program.stage == ShaderType::TessellationControl) {
@@ -134,7 +138,9 @@ void DefineTessellationExecutionModes(EmitterState& state) {
 		                               tess.output_control_points);
 	} else {
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeTriangles);
-		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSpacingFractionalOdd);
+		state.builder.AddExecutionMode(
+		    state.main_func, tess.partitioning == 0u ? spv::ExecutionModeSpacingEqual
+		                                             : spv::ExecutionModeSpacingFractionalOdd);
 		state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeVertexOrderCw);
 	}
 }

tests/ShaderRecompilerComputeTests.cpp

@@ -43885,7 +43885,8 @@ void CheckTessellationProgram(const char *name, u32 ls_stride, u32 hs_stride) {
 
 // PPSA25468: CP0 writes patch constants below the 32 KiB offchip ring end
 // through mul24 addresses that reuse data VGPRs under nested EXEC masks, and
-// the TES reads them back.
+// the TES reads them back. As in God of War Ragnarok, the HS reads its next LS
+// record, loops on a v_cmpx header and CP0 reads back another CP's outputs.
 void CheckTessellationPatchConstants() {
   using namespace ShaderRecompiler;
   using Attribute = IR::TessellationAttribute;
@@ -43899,7 +43900,7 @@ void CheckTessellationPatchConstants() {
   local.push_back(EncodeDs1(0, 2, 20));
   local.push_back(EncodeSop1(0x20, 0, 6));
 
-  const auto make_control = [&](bool divergent) {
+  const auto make_control = [&](bool divergent, bool carried) {
     std::vector<u32> control;
     AppendVop3(&control, 0x365, 0, 193u, InlineU32(0));
     AppendVop3(&control, 0x366, 0, 193u, Vgpr(0));
@@ -43912,7 +43913,10 @@ void CheckTessellationPatchConstants() {
     control.push_back(0xffu);
     control.push_back(EncodeVop2(0x0b, 0, 255u, 16)); // v_mul_u32_u24 v0, hs_stride, v16
     control.push_back(hs_stride);
-    AppendVop3(&control, 0x169, 1, Vgpr(16), 255u); // v_mul_lo_u32 v1, v16, ls_stride
+    control.push_back(EncodeVopc(0xc4, InlineU32(2), 16));    // v_cmp_gt_u32 vcc, 2, v16
+    control.push_back(EncodeVop2(0x25, 1, InlineU32(1), 16)); // v_add_nc_u32 v1, 1, v16
+    control.push_back(EncodeVop2(0x01, 1, InlineU32(0), 1));  // v_cndmask_b32 v1, 0, v1, vcc
+    AppendVop3(&control, 0x169, 1, Vgpr(1), 255u); // v_mul_lo_u32 v1, v1, ls_stride
     control.push_back(ls_stride);
     AppendVop3(&control, 0x143, 17, 255u, Vgpr(50), Vgpr(0));
     control.push_back(3 * hs_stride);
@@ -43932,12 +43936,33 @@ void CheckTessellationPatchConstants() {
       control.push_back(EncodeVop2(0x25, 17, InlineU32(4), 17)); // v_add_nc_u32 v17, 4, v17
     }
     control[skip_scale] = EncodeSopp(0x06, control.size() - skip_scale - 1u);
+    control.push_back(EncodeSop1(0x04, 6, 126));       // s_mov_b64 s[6:7], exec
+    control.push_back(EncodeSMovB32(5, InlineU32(0))); // s_mov_b32 s5, 0
+    const auto loop = control.size();
+    control.push_back(EncodeVopc(0xd1, 5, 16)); // v_cmpx_lt_u32 s5, v16
+    const auto exit_loop = control.size();
+    control.push_back(0u); // s_cbranch_execz
     control.push_back(EncodeMubuf0(0x1c, 24));
     control.push_back(EncodeMubuf1(12, 2, 17, 2));
+    // Only the back edge shows a loop-carried v17 at the store above.
+    if (carried) {
+      control.push_back(EncodeVop2(0x25, 17, InlineU32(4), 17)); // v_add_nc_u32 v17, 4, v17
+    }
+    control.push_back(EncodeVop2(0x25, 13, InlineU32(1), 13)); // v_add_nc_u32 v13, 1, v13
+    control.push_back(EncodeSop2(0x00, 5, 5, InlineU32(1)));   // s_add_u32 s5, s5, 1
+    control.push_back(EncodeSopp(0x02, loop - control.size() - 1u)); // s_branch
+    control[exit_loop] = EncodeSopp(0x08, control.size() - exit_loop - 1u);
+    control.push_back(EncodeSop1(0x04, 126, 6)); // s_mov_b64 exec, s[6:7]
+    control.push_back(EncodeSopp(0x0a));         // s_barrier
     // CP0 reuses data VGPR v10 for the ring address 0x8000 - (patch + 1) * 0xf0.
     control.push_back(EncodeVopc(0xd4, InlineU32(1), 16)); // v_cmpx_gt_u32 1, v16
     const auto skip_patch = control.size();
     control.push_back(0u); // s_cbranch_execz
+    control.push_back(EncodeVop2(0x0b, 11, 255u, 50)); // v_mul_u32_u24 v11, 3 * hs_stride, v50
+    control.push_back(3 * hs_stride);
+    // buffer_load_dwordx2 v[11:12], v11, s[8:11], s2 offen offset:hs_stride + 16 (CP1)
+    control.push_back(EncodeMubuf0(0x0d, hs_stride + 16));
+    control.push_back(EncodeMubuf1(11, 2, 11, 2));
     control.push_back(EncodeVop2(0x25, 10, InlineU32(1), 50)); // v_add_nc_u32 v10, 1, v50
     control.push_back(EncodeVop2(0x0b, 10, 255u, 10));         // v_mul_u32_u24 v10, 0xf0, v10
     control.push_back(patch_stride);
@@ -43959,7 +43984,7 @@ void CheckTessellationPatchConstants() {
     AppendEnd(&control);
     return control;
   };
-  const auto control = make_control(false);
+  const auto control = make_control(false, false);
 
   std::vector<u32> evaluation;
   AppendVop3(&evaluation, 0x169, 24, 255u, Vgpr(7));
@@ -43982,13 +44007,14 @@ void CheckTessellationPatchConstants() {
   ShaderTessellationInputInfo tess{.input_control_points = 3,
                                   .output_control_points = 3,
                                   .domain = 1,
-                                  .partitioning = 2,
+                                  .partitioning = 0,
                                   .output_topology = 2};
   AnalyzeTessellationPrograms(local, control, tess);
   Require(name, "reflected interface",
           tess.ls_stride == ls_stride && tess.hs_stride == hs_stride &&
               tess.patch_base == 0x7f10u && tess.patch_size == 28u,
-          "b128 LS records, joined HS paths and CP0 patch stores must reflect the layout");
+          "b128 LS records, selected LS reads, joined and looping HS paths and CP0 patch "
+          "stores must reflect the layout");
 
   auto ls = CompileTessellationStage(name, local, ShaderType::Local, tess);
   auto hs = CompileTessellationStage(name, control, ShaderType::TessellationControl, tess);
@@ -43999,6 +44025,10 @@ void CheckTessellationPatchConstants() {
   Require(name, "predicated patch outputs",
           hs.addresses[Attribute::PatchOutput] == std::set<u32>{0x7f10, 0x7f20, 0x7f24, 0x7f28},
           "CP0 mul24 addresses under nested EXEC masks must reach patch constants");
+  Require(name, "control point read-back",
+          hs.addresses[Attribute::ControlOutput] == std::set<u32>{hs_stride + 16, hs_stride + 20} &&
+              hs.assembly.find("OpUDiv %uint %uint_96 %uint_80") != std::string::npos,
+          "CP0 must read CP1's outputs by address instead of its own invocation's");
   Require(name, "patch inputs",
           ds.addresses[Attribute::PatchInput] == std::set<u32>{0x7f20, 0x7f24} &&
               ds.addresses[Attribute::EvaluationInput] ==
@@ -44008,12 +44038,16 @@ void CheckTessellationPatchConstants() {
           HasPatchInterface(hs.assembly, "Output", 2, 5) &&
               HasPatchInterface(ds.assembly, "Input", 2, 5),
           "HS and TES patch variables must share their Location and type");
+  Require(name, "integer partitioning", ds.assembly.find("SpacingEqual") != std::string::npos,
+          "integer partitioning must map to equal spacing");
   std::printf("[host]    %-32s ok\n", name);
 #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
-  ExpectFatal("TessellationDivergentJoin", [&] {
+  for (const bool carried : {false, true}) {
+    ExpectFatal(carried ? "TessellationLoopCarriedAddress" : "TessellationDivergentJoin", [&] {
       ShaderTessellationInputInfo info{.input_control_points = 3, .output_control_points = 3};
-    AnalyzeTessellationPrograms(local, make_control(true), info);
+      AnalyzeTessellationPrograms(local, make_control(!carried, carried), info);
     });
+  }
 #endif
 }