shader: correct DS float min/max operands and special values

Co-authored-by: shadowbeat070 <70843316+shadowbeat070@users.noreply.github.com>
This commit is contained in:
nmzik
2026-10-05 03:56:42 +04:00
parent d8be245c22
commit e85279ea7b
8 changed files with 179 additions and 82 deletions
@@ -463,6 +463,9 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind);
uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
bool max_value);
uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
bool max_value);
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
inline constexpr auto EmitSelectValueU32 =
@@ -1227,34 +1227,10 @@ uint32_t EmitBufferFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto& mem = ctx.Memory(inst);
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
const auto index = DwordIndex(ctx, inst, mem);
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
ctx.state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable,
ctx.Arg(inst, 1));
const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), data,
ctx.scratch_u32_variable);
AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, resource, index),
mem.kind, [&](uint32_t old) {
const auto old_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
const auto compare_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data);
const auto compare = Binary(
ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f);
return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
});
});
});
EmitAtomicUpdate(ctx, inst, mem,
[max_value](EmitterState& state, uint32_t old, uint32_t value) {
return EmitDsFloatAtomicReplacement(state, old, value, max_value);
});
}
uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -438,6 +438,20 @@ void EmitAtomicMemoryBarrier(EmitterState& state, IR::ResourceKind kind) {
ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory));
}
namespace {
uint32_t FloatOrderKey(EmitterState& state, uint32_t bits) {
const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
EmitAndConstant(state, bits, 0x80000000u), 0u);
const auto negative_key = state.builder.AllocateId();
state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
const auto positive_key =
EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
return EmitSelectValueU32(state, negative, negative_key, positive_key);
}
} // namespace
uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
bool max_value) {
struct OrderedBits {
@@ -447,15 +461,7 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
};
const auto classify = [&](uint32_t bits) {
const auto cls = EmitClassifyF32Bits(state, bits);
const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
EmitAndConstant(state, bits, 0x80000000u), 0u);
const auto negative_key = state.builder.AllocateId();
state.builder.AddFunction(spv::OpNot, TypeU32(state), negative_key, bits);
const auto positive_key =
EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
return OrderedBits {
cls.nan, cls.zero,
EmitSelectValueU32(state, negative, negative_key, positive_key)};
return OrderedBits {cls.nan, cls.zero, FloatOrderKey(state, bits)};
};
const auto source_class = classify(source);
const auto old_class = classify(old);
@@ -470,6 +476,45 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
old);
}
uint32_t EmitDsFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
bool max_value) {
struct OrderedBits {
uint32_t nan;
uint32_t signaling_nan;
uint32_t key;
};
const auto classify = [&](uint32_t bits) {
const auto abs = EmitAndConstant(state, bits, 0x7fffffffu);
const auto nan = EmitCompareU32Constant(state, spv::OpUGreaterThan, abs, 0x7f800000u);
const auto signaling_nan = EmitLogicalAndBool(
state, nan, EmitCompareU32Constant(state, spv::OpIEqual,
EmitAndConstant(state, bits, 0x00400000u), 0u));
const auto small = EmitCompareU32Constant(state, spv::OpULessThan, abs, 0x00800000u);
const auto sign = EmitAndConstant(state, bits, 0x80000000u);
const auto positive_denorm = EmitLogicalAndBool(
state, EmitCompareU32Constant(state, spv::OpIEqual, sign, 0u),
EmitCompareU32Constant(state, spv::OpINotEqual, abs, 0u));
// Negative denorms tie with -0; positive denorms tie only with each other.
const auto small_bits = EmitSelectValueU32(
state, positive_denorm, ConstantU32(state, 1u), sign);
const auto key_bits = EmitSelectValueU32(state, small, small_bits, bits);
return OrderedBits {nan, signaling_nan, FloatOrderKey(state, key_bits)};
};
const auto lhs = classify(old);
const auto rhs = classify(source);
const auto choose_source = state.builder.AllocateId();
state.builder.AddFunction(max_value ? spv::OpUGreaterThan : spv::OpULessThanEqual,
TypeBool(state), choose_source, rhs.key, lhs.key);
// Keep original bits: ordering denorms does not flush the selected operand.
auto result = EmitSelectValueU32(state, choose_source, source, old);
result = EmitSelectValueU32(state, lhs.nan, source, result);
result = EmitSelectValueU32(state, rhs.nan, old, result);
result = EmitSelectValueU32(state, rhs.signaling_nan,
EmitOrU32(state, source, ConstantU32(state, 0x00400000u)), result);
return EmitSelectValueU32(state, lhs.signaling_nan,
EmitOrU32(state, old, ConstantU32(state, 0x00400000u)), result);
}
uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
if ((control & 0xc000u) == 0xc000u) {
const uint32_t mask = control & 0x1fu;
@@ -689,9 +689,7 @@ void EmitProgram(EmitterState& state) {
DefineBvhIntersect(state);
for (const auto* block: program.blocks) {
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32;
})) {
ctx.scratch_u32_variable = state.builder.AllocateId();
if (state.lane_count == 2) {
@@ -205,9 +205,9 @@ uint32_t DsSourceCount(Opcode opcode) {
case Opcode::DS_WRITE2ST64_B32:
case Opcode::DS_WRITE2_B64:
case Opcode::DS_WRITE2ST64_B64:
case Opcode::DS_MSKOR_B32:
case Opcode::DS_MSKOR_B32: return 3u;
case Opcode::DS_MIN_F32:
case Opcode::DS_MAX_F32: return 3u;
case Opcode::DS_MAX_F32:
case Opcode::DS_PERMUTE_B32:
case Opcode::DS_BPERMUTE_B32: return 2u;
case Opcode::DS_READ_ADDTID_B32:
@@ -280,8 +280,8 @@ VALUE_OPCODE(WriteSharedU32, Void, U32, U32, U1)
VALUE_OPCODE(WriteSharedU32x2, Void, U32, U32, U32, U1)
VALUE_OPCODE(WriteSharedU32x3, Void, U32, U32, U32, U32, U1)
VALUE_OPCODE(WriteSharedU32x4, Void, U32, U32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMin32, Void, U32, U32, U1)
VALUE_OPCODE(SharedAtomicFMax32, Void, U32, U32, U1)
VALUE_OPCODE(SharedAtomicSwap32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicIAdd32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicIAdd64, Void, U32, U64, U1)
+100 -4
View File
@@ -30338,7 +30338,7 @@ TestCase DsMiscVariants() {
return test;
}
TestCase DsFloatMinMaxUsesSeparateCompareOperand() {
TestCase DsFloatMinMaxIgnoresData1() {
using O = ShaderOpcode;
std::vector<u32> code;
@@ -30366,10 +30366,10 @@ TestCase DsFloatMinMaxUsesSeparateCompareOperand() {
AppendEnd(&code);
TestCase test;
test.name = "DsFloatMinMaxUsesSeparateCompareOperand";
test.name = "DsFloatMinMaxIgnoresData1";
test.code = code;
test.initial = std::vector<u32>(2, 0);
test.expected = {0x41100000u, 0x3f800000u};
test.expected = {0x40800000u, 0x40800000u};
test.opcodes = {O::V_MOV_B32, O::DS_WRITE_B32, O::DS_MIN_F32,
O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORD,
O::S_ENDPGM};
@@ -30380,6 +30380,100 @@ TestCase DsFloatMinMaxUsesSeparateCompareOperand() {
return test;
}
TestCase DsFloatMinMaxClasses(bool gds) {
using O = ShaderOpcode;
// Rows are memory, columns source: -inf, -normal, -denorm, -0, +0,
// +denorm, +normal, +inf, qNaN, sNaN. O/N select old/new; Q/R quiet them.
constexpr std::string_view minimum[] = {
"OOOOOOOOOR", "NNOOOOOOOR", "NNNNOOOOOR", "NNNOOOOOOR", "NNNNOOOOOR",
"NNNNNNOOOR", "NNNNNNOOOR", "NNNNNNNOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
constexpr std::string_view maximum[] = {
"ONNNNNNNOR", "OONNNNNNOR", "OOOONNNNOR", "OOOONNNNOR", "OOOOONNNOR",
"OOOOOONNOR", "OOOOOONNOR", "OOOOOOOOOR", "NNNNNNNNOR", "QQQQQQQQQQ"};
constexpr u32 old_values[] = {
0xff800000u, 0xc0000000u, 0x80000100u, 0x80000000u, 0u,
0x00000100u, 0x40000000u, 0x7f800000u, 0x7fc12345u, 0x7f812345u};
constexpr u32 sources[] = {
0xff800000u, 0xc0400000u, 0x80000001u, 0x80000000u, 0u,
0x00000001u, 0x40400000u, 0x7f800000u, 0xffc54321u, 0xff854321u};
const auto select = [](char choice, u32 old, u32 source) {
const u32 bits = choice == 'O' || choice == 'Q' ? old : source;
return choice == 'Q' || choice == 'R' ? bits | 0x00400000u : bits;
};
TestCase test;
test.name = gds ? "DsFloatMinMaxGdsClasses" : "DsFloatMinMaxLdsClasses";
for (u32 row = 0; row < std::size(old_values); ++row) {
for (u32 column = 0; column < std::size(sources); ++column) {
const auto old = old_values[row];
const auto source = sources[column];
test.initial.insert(test.initial.end(), {old, source, 0u, 0u});
test.expected.insert(test.expected.end(),
{old, source, select(minimum[row][column], old, source),
select(maximum[row][column], old, source)});
}
}
// Reversing positive denorm magnitudes still selects source for min and old for max.
test.initial.insert(test.initial.end(), {1u, 0x100u, 0u, 0u});
test.expected.insert(test.expected.end(), {1u, 0x100u, 0x100u, 1u});
const auto count = static_cast<u32>(test.initial.size() / 4u);
auto &code = test.code;
code.push_back(EncodeVop2(0x1a, 1, InlineU32(3), 0));
code.push_back(EncodeVop2(0x1a, 8, InlineU32(4), 0));
code.push_back(EncodeMubuf0(0x0d));
code.push_back(EncodeMubuf1(2, 0, 8));
for (u32 offset : {0u, 4u}) {
code.push_back(EncodeDs0(0x0d, offset, gds));
code.push_back(EncodeDs1(0, 2, 1));
}
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1Ex(0, 250, 3, 1)); // DATA1 is unused and uninitialized.
code.push_back(EncodeDs0(0x13, 4, gds));
code.push_back(EncodeDs1Ex(0, 251, 3, 1));
AppendVMovLiteral(&code, 6, 0xff800000u);
AppendVMovLiteral(&code, 7, 0x7f800000u);
code.push_back(EncodeSop1(0x04, 4, 126));
code.push_back(EncodeSop1(0x04, 126, InlineU32(0)));
// Inactive updates must leave the table results untouched.
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1(0, 6, 1));
code.push_back(EncodeDs0(0x13, 4, gds));
code.push_back(EncodeDs1(0, 7, 1));
code.push_back(EncodeSop1(0x04, 126, 4));
// Active accesses immediately beyond the allocation must also be discarded.
AppendVMovU32(&code, 9, count * 8u);
code.push_back(EncodeDs0(0x12, 0, gds));
code.push_back(EncodeDs1(0, 6, 9));
code.push_back(EncodeDs0(0x13, 0, gds));
code.push_back(EncodeDs1(0, 7, 9));
for (u32 component = 0; component < 2u; ++component) {
code.push_back(EncodeDs0(0x36, component * 4u, gds));
code.push_back(EncodeDs1(4u + component, 0, 1));
}
code.push_back(EncodeMubuf0(0x1d, 8));
code.push_back(EncodeMubuf1(4, 12, 8));
AppendEnd(&code);
test.compute_info.threads_num[0] = count;
test.compute_info.threads_num[1] = 1;
test.compute_info.threads_num[2] = 1;
test.compute_info.thread_ids_num = 1;
test.compute_info.lds_size_dwords = count * 2u;
test.has_compute_info = true;
test.opcodes = {O::V_MOV_B32, O::V_LSHLREV_B32, O::S_MOV_B64,
O::BUFFER_LOAD_DWORDX2, O::DS_WRITE_B32, O::DS_MIN_F32,
O::DS_MAX_F32, O::DS_READ_B32, O::BUFFER_STORE_DWORDX2,
O::S_ENDPGM};
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange", "OpLoopMerge"};
test.forbidden_spirv = {"OpFOrdLessThan", "OpFOrdGreaterThan"};
if (gds) {
test.gds_initial.resize(count * 2u);
for (u32 i = 0; i < count; ++i) {
test.expected_gds.push_back(test.expected[i * 4u + 2u]);
test.expected_gds.push_back(test.expected[i * 4u + 3u]);
}
}
return test;
}
TestCase DsSwizzleInvalidSourceLaneZero() {
using O = ShaderOpcode;
@@ -34528,7 +34622,9 @@ std::vector<TestCase> MakeCases() {
}
}
AddCase(DsMiscVariants);
AddCase(DsFloatMinMaxUsesSeparateCompareOperand);
AddCase(DsFloatMinMaxIgnoresData1);
AddCase([] { return DsFloatMinMaxClasses(false); });
AddCase([] { return DsFloatMinMaxClasses(true); });
AddCase(DsSwizzleInvalidSourceLaneZero);
AddCase(DsPermuteCapturedExecOffsetAndWrap);
AddCase(DsPermuteWave64UsesIndependentHalves);
+13 -34
View File
@@ -7741,40 +7741,18 @@ void TestNewShaderRecompilerDsAddtidTranslation() {
CheckSpirvBinaryValidates(result.spirv);
}
void TestNewShaderRecompilerDsFloatMinMaxTranslation() {
const uint32_t shader[] = {
EncodeDs0(0x12, 4), EncodeDs1Ex(0, 9, 7, 1), // ds_min_f32 v7, v9, v1
EncodeDs0(0x13, 8), EncodeDs1Ex(0, 10, 8, 1), // ds_max_f32 v8, v10, v1
0xbf810000u,
};
auto options = MakeCompileOptions(ShaderType::Compute);
options.dump_ir = true;
auto result = RecompileForTest(shader, options);
Check((result.decoded_dump.find("ds_min_f32") != std::string::npos),
"new decoder did not decode DS float min");
Check((result.decoded_dump.find("ds_max_f32") != std::string::npos),
"new decoder did not decode DS float max");
Check((result.ir_dump.find("DsMinF32 null, v7, v1") != std::string::npos),
"DS float min did not lower to explicit IR");
Check((result.ir_dump.find("DsMaxF32 null, v8, v1") != std::string::npos),
"DS float max did not lower to explicit IR");
Check((result.ir_dump.find("v9") != std::string::npos),
"DS float min did not retain DATA1 compare operand");
Check((result.ir_dump.find("v10") != std::string::npos),
"DS float max did not retain DATA1 compare operand");
Check(SpirvContainsOpcode(result.spirv, 12),
"SPIR-V binary does not contain OpExtInst");
Check(SpirvContainsOpcode(result.spirv, 61),
"SPIR-V binary does not contain OpLoad");
Check(SpirvContainsOpcode(result.spirv, 62),
"SPIR-V binary does not contain OpStore");
Check(SpirvContainsOpcode(result.spirv, 65),
"SPIR-V binary does not contain OpAccessChain");
Check(SpirvContainsOpcode(result.spirv, 124),
"SPIR-V binary does not contain OpBitcast");
CheckSpirvBinaryValidates(result.spirv);
void TestDsFloatMinMaxDecoder() {
using namespace ShaderRecompiler::Decoder;
for (bool max_value : {false, true}) {
const uint32_t shader[] = {
EncodeDs0(max_value ? 0x13 : 0x12, 4), EncodeDs1Ex(0, 250, 7, 1)};
Instruction decoded;
DecodeInstruction(shader, 0u, decoded);
Check(decoded.opcode == (max_value ? Opcode::DS_MAX_F32 : Opcode::DS_MIN_F32) &&
decoded.src_count == 2u && decoded.src0.reg == 1u &&
decoded.src1.reg == 7u && decoded.offset == 4u,
"DS float min/max must read only address and DATA0");
}
}
void TestNewShaderRecompilerCfgStraightLine() {
@@ -14805,6 +14783,7 @@ int main() {
// here.
TestScalarAshrI64Decoder();
TestImageAtomicWidthDecoder();
TestDsFloatMinMaxDecoder();
TestNewShaderDecoderArchitecture();
TestImageAddressOperands();
TestSopkCompareImmediateExtension();