shader_recompiler: add SSA

This commit is contained in:
nmzik
2026-08-15 03:11:19 +02:00
parent bc2f077c37
commit b42380ced6
113 changed files with 40415 additions and 32616 deletions
@@ -0,0 +1,250 @@
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <array>
#include <set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint32_t MaxPushConstantBytes = 128;
constexpr std::array ImageBindingKinds = {
DescriptorBindingKind::Sampled1D,
DescriptorBindingKind::Sampled1DArray,
DescriptorBindingKind::Sampled2D,
DescriptorBindingKind::Sampled2DArray,
DescriptorBindingKind::Sampled2DMsaa,
DescriptorBindingKind::Sampled2DMsaaArray,
DescriptorBindingKind::Sampled3D,
DescriptorBindingKind::SampledUint1D,
DescriptorBindingKind::SampledUint1DArray,
DescriptorBindingKind::SampledUint2D,
DescriptorBindingKind::SampledUint2DArray,
DescriptorBindingKind::SampledUint2DMsaa,
DescriptorBindingKind::SampledUint2DMsaaArray,
DescriptorBindingKind::SampledUint3D,
DescriptorBindingKind::Storage1D,
DescriptorBindingKind::Storage1DArray,
DescriptorBindingKind::Storage2D,
DescriptorBindingKind::Storage2DArray,
DescriptorBindingKind::Storage3D,
DescriptorBindingKind::StorageUint1D,
DescriptorBindingKind::StorageUint1DArray,
DescriptorBindingKind::StorageUint2D,
DescriptorBindingKind::StorageUint2DArray,
DescriptorBindingKind::StorageUint3D,
};
bool ImageBinding(const ImageResource& image, DescriptorBindingKind& result) {
using Dimension = Decoder::ImageDimension;
using Kind = DescriptorBindingKind;
switch (image.kind) {
case ResourceKind::Image:
switch (image.dimension) {
case Dimension::Dim1D: result = Kind::Sampled1D; return true;
case Dimension::Dim1DArray: result = Kind::Sampled1DArray; return true;
case Dimension::Dim2D: result = Kind::Sampled2D; return true;
case Dimension::Dim2DArray: result = Kind::Sampled2DArray; return true;
case Dimension::Dim2DMsaa: result = Kind::Sampled2DMsaa; return true;
case Dimension::Dim2DMsaaArray: result = Kind::Sampled2DMsaaArray; return true;
case Dimension::Dim3D: result = Kind::Sampled3D; return true;
default: return false;
}
case ResourceKind::ImageUint:
switch (image.dimension) {
case Dimension::Dim1D: result = Kind::SampledUint1D; return true;
case Dimension::Dim1DArray: result = Kind::SampledUint1DArray; return true;
case Dimension::Dim2D: result = Kind::SampledUint2D; return true;
case Dimension::Dim2DArray: result = Kind::SampledUint2DArray; return true;
case Dimension::Dim2DMsaa: result = Kind::SampledUint2DMsaa; return true;
case Dimension::Dim2DMsaaArray: result = Kind::SampledUint2DMsaaArray; return true;
case Dimension::Dim3D: result = Kind::SampledUint3D; return true;
default: return false;
}
case ResourceKind::StorageImage:
switch (image.dimension) {
case Dimension::Dim1D: result = Kind::Storage1D; return true;
case Dimension::Dim1DArray: result = Kind::Storage1DArray; return true;
case Dimension::Dim2D: result = Kind::Storage2D; return true;
case Dimension::Dim2DArray: result = Kind::Storage2DArray; return true;
case Dimension::Dim3D: result = Kind::Storage3D; return true;
default: return false;
}
case ResourceKind::StorageImageUint:
switch (image.dimension) {
case Dimension::Dim1D: result = Kind::StorageUint1D; return true;
case Dimension::Dim1DArray: result = Kind::StorageUint1DArray; return true;
case Dimension::Dim2D: result = Kind::StorageUint2D; return true;
case Dimension::Dim2DArray: result = Kind::StorageUint2DArray; return true;
case Dimension::Dim3D: result = Kind::StorageUint3D; return true;
default: return false;
}
default: return false;
}
}
bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
std::set<uint32_t> registers;
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::GetUserData || !inst.HasUses()) {
continue;
}
if (inst.Arg(0).GetType() != Type::ScalarReg) {
return false;
}
const auto index = RegIndex(inst.Arg(0).ScalarRegister());
if (index >= NumScalarRegs) {
return false;
}
registers.insert(index);
}
}
result.assign(registers.begin(), registers.end());
return true;
}
void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
std::vector<uint32_t> resources = {}) {
layout.descriptors.push_back(
{kind, static_cast<uint32_t>(layout.descriptors.size()), std::move(resources)});
}
bool UsesGds(const Program& program) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == ValueOpcode::GetGdsResource && inst.HasUses()) {
return true;
}
for (size_t index = 0; index < inst.NumArgs(); index++) {
if (inst.Arg(index).GetType() == Type::GdsResource) {
return true;
}
}
}
}
return false;
}
} // namespace
bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std::string* error) {
if (!program.shader_info_complete || program.binding_layout_complete) {
if (error != nullptr) {
*error = !program.shader_info_complete ? "shader info is not ready"
: "binding layout already allocated";
}
return false;
}
if (options.push_constant_offset > MaxPushConstantBytes) {
if (error != nullptr) {
*error = "push-constant offset exceeds the Vulkan minimum guarantee";
}
return false;
}
if (options.push_constant_offset % 4u != 0) {
if (error != nullptr) {
*error = "push-constant offset is not dword aligned";
}
return false;
}
if (program.values == nullptr) {
if (error != nullptr) {
*error = "typed value program is not available";
}
return false;
}
BindingLayout next;
next.descriptor_set = options.descriptor_set;
next.push_constant_offset = options.push_constant_offset;
if (!CollectUserData(program, next.user_data_registers)) {
if (error != nullptr) {
*error = "typed shader contains an invalid user-data register";
}
return false;
}
next.buffer_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
next.buffer_offset_count = static_cast<uint32_t>(program.info.buffers.size());
if (!program.info.buffers.empty()) {
std::vector<uint32_t> resources(program.info.buffers.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(next, DescriptorBindingKind::Buffers, std::move(resources));
}
std::array<std::vector<uint32_t>, ImageBindingKinds.size()> image_groups;
for (uint32_t i = 0; i < program.info.images.size(); i++) {
DescriptorBindingKind kind;
if (!ImageBinding(program.info.images[i], kind)) {
if (error != nullptr) {
*error = "shader info contains an invalid image binding class";
}
return false;
}
const auto group = std::find(ImageBindingKinds.begin(), ImageBindingKinds.end(), kind);
if (group == ImageBindingKinds.end()) {
if (error != nullptr) {
*error = "shader info contains an unmapped image binding class";
}
return false;
}
image_groups[static_cast<size_t>(group - ImageBindingKinds.begin())].push_back(i);
}
for (uint32_t i = 0; i < image_groups.size(); i++) {
if (!image_groups[i].empty()) {
AddBinding(next, ImageBindingKinds[i], std::move(image_groups[i]));
}
}
if (!program.info.samplers.empty()) {
std::vector<uint32_t> resources(program.info.samplers.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
}
if (UsesGds(program)) {
AddBinding(next, DescriptorBindingKind::Gds);
}
if (!program.info.addresses.empty()) {
std::vector<uint32_t> resources(program.info.addresses.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
}
if (!program.values->srt_reads.empty()) {
AddBinding(next, DescriptorBindingKind::FlattenedSrt);
}
const auto available_push_dwords = (MaxPushConstantBytes - options.push_constant_offset) / 4u;
const auto push_limit = std::min(options.max_push_dwords, available_push_dwords);
if (next.ShaderDataDwords() <= push_limit) {
next.push_constant_size = next.ShaderDataDwords() * sizeof(uint32_t);
} else {
AddBinding(next, DescriptorBindingKind::UserData);
}
program.bindings = std::move(next);
program.binding_layout_complete = true;
return true;
}
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind) {
for (const auto& binding: layout.descriptors) {
if (binding.kind == kind) {
return &binding;
}
}
return nullptr;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,20 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct BindingLayoutOptions {
uint32_t descriptor_set = 0;
uint32_t push_constant_offset = 0;
uint32_t max_push_dwords = 32;
};
bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std::string* error);
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ */
@@ -0,0 +1,612 @@
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
#include <algorithm>
#include <bit>
#include <cstdint>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
Value Arg(const Inst& inst, size_t index) {
return inst.Arg(index).Resolve();
}
bool IsImmediate(Value value, Type type) {
return value.IsImmediate() && value.GetType() == type;
}
void Replace(Inst& inst, Value value) {
inst.ReplaceUsesWith(value.Resolve());
}
template <typename Function>
bool FoldU32(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U32) || !IsImmediate(rhs, Type::U32)) {
return false;
}
Replace(inst, Value(static_cast<uint32_t>(function(lhs.U32(), rhs.U32()))));
return true;
}
template <typename Function>
bool FoldU64(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U64) || !IsImmediate(rhs, Type::U64)) {
return false;
}
Replace(inst, Value(static_cast<uint64_t>(function(lhs.U64(), rhs.U64()))));
return true;
}
template <typename Function>
bool FoldU64Shift(Inst& inst, Function function) {
const auto value = Arg(inst, 0);
const auto shift = Arg(inst, 1);
if (!IsImmediate(value, Type::U64) || !IsImmediate(shift, Type::U32)) {
return false;
}
Replace(inst, Value(static_cast<uint64_t>(function(value.U64(), shift.U32()))));
return true;
}
template <typename Function>
bool FoldU32Compare(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U32) || !IsImmediate(rhs, Type::U32)) {
return false;
}
Replace(inst, Value(function(lhs.U32(), rhs.U32())));
return true;
}
template <typename Function>
bool FoldU64Compare(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U64) || !IsImmediate(rhs, Type::U64)) {
return false;
}
Replace(inst, Value(function(lhs.U64(), rhs.U64())));
return true;
}
template <typename Function>
bool FoldLogical(Inst& inst, Function function) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (!IsImmediate(lhs, Type::U1) || !IsImmediate(rhs, Type::U1)) {
return false;
}
Replace(inst, Value(function(lhs.U1(), rhs.U1())));
return true;
}
bool ReplaceBinaryIdentity(Inst& inst, Type type, uint64_t identity) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, type) &&
(type == Type::U32 ? lhs.U32() == identity : lhs.U64() == identity)) {
Replace(inst, rhs);
return true;
}
if (IsImmediate(rhs, type) &&
(type == Type::U32 ? rhs.U32() == identity : rhs.U64() == identity)) {
Replace(inst, lhs);
return true;
}
return false;
}
bool FoldSelect(Inst& inst) {
const auto condition = Arg(inst, 0);
const auto true_value = Arg(inst, 1);
const auto false_value = Arg(inst, 2);
if (IsImmediate(condition, Type::U1)) {
Replace(inst, condition.U1() ? true_value : false_value);
return true;
}
if (true_value == false_value) {
Replace(inst, true_value);
return true;
}
return false;
}
bool FoldPhi(Inst& inst) {
Value same;
for (size_t index = 0; index < inst.NumArgs(); index++) {
const auto value = Arg(inst, index);
if (value.TryInstruction() == &inst) {
continue;
}
if (same.IsEmpty()) {
same = value;
} else if (same != value) {
return false;
}
}
if (same.IsEmpty()) {
return false;
}
Replace(inst, same);
return true;
}
bool FoldBitCast(Inst& inst, ValueOpcode reverse) {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::F32) && inst.GetOpcode() == ValueOpcode::BitCastU32F32) {
Replace(inst, Value(std::bit_cast<uint32_t>(value.F32Value())));
return true;
}
if (IsImmediate(value, Type::U32) && inst.GetOpcode() == ValueOpcode::BitCastF32U32) {
Replace(inst, Value::F32(std::bit_cast<float>(value.U32())));
return true;
}
if (IsImmediate(value, Type::F16) && inst.GetOpcode() == ValueOpcode::BitCastU16F16) {
Replace(inst, Value(value.F16Bits()));
return true;
}
if (IsImmediate(value, Type::U16) && inst.GetOpcode() == ValueOpcode::BitCastF16U16) {
Replace(inst, Value::F16(value.U16()));
return true;
}
if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == reverse) {
Replace(inst, producer->Arg(0));
return true;
}
return false;
}
bool FoldCompositeExtract(Inst& inst, ValueOpcode construct, size_t components) {
const auto composite = Arg(inst, 0);
const auto index = Arg(inst, 1);
if (!IsImmediate(index, Type::U32) || index.U32() >= components) {
return false;
}
const auto component = index.U32();
const auto* producer = composite.TryInstruction();
if (producer != nullptr && producer->GetOpcode() == construct) {
Replace(inst, producer->Arg(component));
return true;
}
if (component < 2u && producer != nullptr &&
producer->GetOpcode() == ValueOpcode::IAddCarry32) {
const auto lhs = producer->Arg(0).Resolve();
const auto rhs = producer->Arg(1).Resolve();
if (IsImmediate(lhs, Type::U32) && IsImmediate(rhs, Type::U32)) {
const auto sum = static_cast<uint64_t>(lhs.U32()) + rhs.U32();
Replace(inst, Value(component == 0u ? static_cast<uint32_t>(sum)
: static_cast<uint32_t>(sum >> 32u)));
return true;
}
}
if (component < 2u && producer != nullptr &&
producer->GetOpcode() == ValueOpcode::UnpackUint2x32) {
const auto packed = producer->Arg(0).Resolve();
if (IsImmediate(packed, Type::U64)) {
Replace(inst, Value(static_cast<uint32_t>(packed.U64() >> (component * 32u))));
return true;
}
}
return false;
}
void FoldInstruction(Inst& inst) {
switch (inst.GetOpcode()) {
case ValueOpcode::Phi: FoldPhi(inst); return;
case ValueOpcode::SelectU1:
case ValueOpcode::SelectU32:
case ValueOpcode::SelectF32: FoldSelect(inst); return;
case ValueOpcode::BitFieldInsert: {
const auto base = Arg(inst, 0);
const auto insert = Arg(inst, 1);
const auto offset = Arg(inst, 2);
const auto count = Arg(inst, 3);
if (IsImmediate(base, Type::U32) && IsImmediate(insert, Type::U32) &&
IsImmediate(offset, Type::U32) && IsImmediate(count, Type::U32) &&
offset.U32() <= 32u && count.U32() <= 32u - offset.U32()) {
if (count.U32() == 0u) {
Replace(inst, base);
return;
}
const auto mask = count.U32() == 32u
? UINT32_MAX
: ((uint32_t {1} << count.U32()) - 1u) << offset.U32();
Replace(inst,
Value((base.U32() & ~mask) | ((insert.U32() << offset.U32()) & mask)));
}
return;
}
case ValueOpcode::BitFieldUExtract:
case ValueOpcode::BitFieldSExtract: {
const auto value = Arg(inst, 0);
const auto offset = Arg(inst, 1);
const auto count = Arg(inst, 2);
if (!IsImmediate(value, Type::U32) || !IsImmediate(offset, Type::U32) ||
!IsImmediate(count, Type::U32) || offset.U32() > 32u ||
count.U32() > 32u - offset.U32()) {
return;
}
if (count.U32() == 0u) {
Replace(inst, Value(0u));
} else if (inst.GetOpcode() == ValueOpcode::BitFieldUExtract) {
const auto mask =
count.U32() == 32u ? UINT32_MAX : (uint32_t {1} << count.U32()) - 1u;
Replace(inst, Value((value.U32() >> offset.U32()) & mask));
} else {
const auto left = 32u - offset.U32() - count.U32();
const auto bits = value.U32() << left;
Replace(inst, Value(static_cast<uint32_t>(std::bit_cast<int32_t>(bits) >>
(left + offset.U32()))));
}
return;
}
case ValueOpcode::BitCastU16F16: FoldBitCast(inst, ValueOpcode::BitCastF16U16); return;
case ValueOpcode::BitCastF16U16: FoldBitCast(inst, ValueOpcode::BitCastU16F16); return;
case ValueOpcode::BitCastU32F32: FoldBitCast(inst, ValueOpcode::BitCastF32U32); return;
case ValueOpcode::BitCastF32U32: FoldBitCast(inst, ValueOpcode::BitCastU32F32); return;
case ValueOpcode::ConvertU16U32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(static_cast<uint16_t>(value.U32())));
}
return;
}
case ValueOpcode::ConvertU32U16: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U16)) {
Replace(inst, Value(static_cast<uint32_t>(value.U16())));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::ConvertU16U32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::ConvertU8U32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(static_cast<uint8_t>(value.U32())));
}
return;
}
case ValueOpcode::ConvertU32U8: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U8)) {
Replace(inst, Value(static_cast<uint32_t>(value.U8())));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::ConvertU8U32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::CompositeExtractU32x2:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x2, 2);
return;
case ValueOpcode::CompositeExtractU32x4:
FoldCompositeExtract(inst, ValueOpcode::CompositeConstructU32x4, 4);
return;
case ValueOpcode::PackUint2x32: {
const auto value = Arg(inst, 0);
const auto* producer = value.TryInstruction();
if (producer != nullptr && producer->GetOpcode() == ValueOpcode::UnpackUint2x32) {
Replace(inst, producer->Arg(0));
} else if (producer != nullptr &&
producer->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
const auto low = producer->Arg(0).Resolve();
const auto high = producer->Arg(1).Resolve();
if (IsImmediate(low, Type::U32) && IsImmediate(high, Type::U32)) {
Replace(inst, Value(static_cast<uint64_t>(low.U32()) |
(static_cast<uint64_t>(high.U32()) << 32u)));
}
}
return;
}
case ValueOpcode::UnpackUint2x32: {
const auto value = Arg(inst, 0);
const auto* producer = value.TryInstruction();
if (producer != nullptr && producer->GetOpcode() == ValueOpcode::PackUint2x32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::IAdd32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a + b; })) {
ReplaceBinaryIdentity(inst, Type::U32, 0u);
}
return;
case ValueOpcode::IAdd64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a + b; })) {
ReplaceBinaryIdentity(inst, Type::U64, 0u);
}
return;
case ValueOpcode::ISub32: {
if (FoldU32(inst, [](uint32_t a, uint32_t b) { return a - b; })) {
return;
}
const auto rhs = Arg(inst, 1);
if (IsImmediate(rhs, Type::U32) && rhs.U32() == 0u) {
Replace(inst, Arg(inst, 0));
}
return;
}
case ValueOpcode::ISub64: {
if (FoldU64(inst, [](uint64_t a, uint64_t b) { return a - b; })) {
return;
}
const auto rhs = Arg(inst, 1);
if (IsImmediate(rhs, Type::U64) && rhs.U64() == 0u) {
Replace(inst, Arg(inst, 0));
}
return;
}
case ValueOpcode::IMul32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a * b; })) {
ReplaceBinaryIdentity(inst, Type::U32, 1u);
}
return;
case ValueOpcode::IMul64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a * b; })) {
ReplaceBinaryIdentity(inst, Type::U64, 1u);
}
return;
case ValueOpcode::UDiv32: {
const auto rhs = Arg(inst, 1);
if (IsImmediate(rhs, Type::U32) && rhs.U32() == 1u) {
Replace(inst, Arg(inst, 0));
} else if (IsImmediate(rhs, Type::U32) && rhs.U32() != 0u) {
FoldU32(inst, [](uint32_t a, uint32_t b) { return a / b; });
}
return;
}
case ValueOpcode::SMulHi:
FoldU32(inst, [](uint32_t a, uint32_t b) {
const auto product = static_cast<int64_t>(std::bit_cast<int32_t>(a)) *
static_cast<int64_t>(std::bit_cast<int32_t>(b));
return static_cast<uint32_t>(static_cast<uint64_t>(product) >> 32u);
});
return;
case ValueOpcode::UMulHi:
FoldU32(inst, [](uint32_t a, uint32_t b) {
return static_cast<uint32_t>((static_cast<uint64_t>(a) * b) >> 32u);
});
return;
case ValueOpcode::IAbs32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst,
Value((value.U32() & 0x80000000u) != 0u ? 0u - value.U32() : value.U32()));
}
return;
}
case ValueOpcode::ShiftLeftLogical32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a << (b & 31u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightLogical32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a >> (b & 31u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 31u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightArithmetic32:
FoldU32(inst, [](uint32_t a, uint32_t b) {
return static_cast<uint32_t>(std::bit_cast<int32_t>(a) >> (b & 31u));
});
return;
case ValueOpcode::ShiftLeftLogical64:
if (!FoldU64Shift(inst, [](uint64_t a, uint32_t b) { return a << (b & 63u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 63u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightLogical64:
if (!FoldU64Shift(inst, [](uint64_t a, uint32_t b) { return a >> (b & 63u); })) {
const auto shift = Arg(inst, 1);
if (IsImmediate(shift, Type::U32) && (shift.U32() & 63u) == 0u) {
Replace(inst, Arg(inst, 0));
}
}
return;
case ValueOpcode::ShiftRightArithmetic64:
FoldU64Shift(inst, [](uint64_t a, uint32_t b) {
return static_cast<uint64_t>(std::bit_cast<int64_t>(a) >> (b & 63u));
});
return;
case ValueOpcode::BitwiseAnd32:
if (!FoldU32(inst, [](uint32_t a, uint32_t b) { return a & b; })) {
ReplaceBinaryIdentity(inst, Type::U32, 0xffffffffu);
}
return;
case ValueOpcode::BitwiseAnd64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a & b; })) {
ReplaceBinaryIdentity(inst, Type::U64, UINT64_MAX);
}
return;
case ValueOpcode::BitwiseOr32:
case ValueOpcode::BitwiseXor32:
if (!FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
return opcode == ValueOpcode::BitwiseOr32 ? a | b : a ^ b;
})) {
ReplaceBinaryIdentity(inst, Type::U32, 0u);
}
return;
case ValueOpcode::BitwiseOr64:
if (!FoldU64(inst, [](uint64_t a, uint64_t b) { return a | b; })) {
ReplaceBinaryIdentity(inst, Type::U64, 0u);
}
return;
case ValueOpcode::BitwiseNot32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(~value.U32()));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::BitwiseNot32) {
Replace(inst, producer->Arg(0));
}
return;
}
case ValueOpcode::BitCount32: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U32)) {
Replace(inst, Value(static_cast<uint32_t>(std::popcount(value.U32()))));
}
return;
}
case ValueOpcode::BitCount64: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U64)) {
Replace(inst, Value(static_cast<uint32_t>(std::popcount(value.U64()))));
}
return;
}
case ValueOpcode::SMin32:
case ValueOpcode::SMax32:
FoldU32(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
const auto lhs = std::bit_cast<int32_t>(a);
const auto rhs = std::bit_cast<int32_t>(b);
return opcode == ValueOpcode::SMin32 ? (lhs < rhs ? a : b) : (lhs > rhs ? a : b);
});
return;
case ValueOpcode::UMin32:
FoldU32(inst, [](uint32_t a, uint32_t b) { return std::min(a, b); });
return;
case ValueOpcode::UMax32:
FoldU32(inst, [](uint32_t a, uint32_t b) { return std::max(a, b); });
return;
case ValueOpcode::IEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a == b; });
return;
case ValueOpcode::INotEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a != b; });
return;
case ValueOpcode::ULessThan32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a < b; });
return;
case ValueOpcode::ULessThanEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a <= b; });
return;
case ValueOpcode::UGreaterThan32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a > b; });
return;
case ValueOpcode::UGreaterThanEqual32:
FoldU32Compare(inst, [](uint32_t a, uint32_t b) { return a >= b; });
return;
case ValueOpcode::SLessThan32:
case ValueOpcode::SLessThanEqual32:
case ValueOpcode::SGreaterThan32:
case ValueOpcode::SGreaterThanEqual32:
FoldU32Compare(inst, [opcode = inst.GetOpcode()](uint32_t a, uint32_t b) {
const auto lhs = std::bit_cast<int32_t>(a);
const auto rhs = std::bit_cast<int32_t>(b);
switch (opcode) {
case ValueOpcode::SLessThan32: return lhs < rhs;
case ValueOpcode::SLessThanEqual32: return lhs <= rhs;
case ValueOpcode::SGreaterThan32: return lhs > rhs;
default: return lhs >= rhs;
}
});
return;
case ValueOpcode::IEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a == b; });
return;
case ValueOpcode::INotEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a != b; });
return;
case ValueOpcode::ULessThan64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a < b; });
return;
case ValueOpcode::ULessThanEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a <= b; });
return;
case ValueOpcode::UGreaterThan64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a > b; });
return;
case ValueOpcode::UGreaterThanEqual64:
FoldU64Compare(inst, [](uint64_t a, uint64_t b) { return a >= b; });
return;
case ValueOpcode::SLessThan64:
case ValueOpcode::SLessThanEqual64:
case ValueOpcode::SGreaterThan64:
case ValueOpcode::SGreaterThanEqual64:
FoldU64Compare(inst, [opcode = inst.GetOpcode()](uint64_t a, uint64_t b) {
const auto lhs = std::bit_cast<int64_t>(a);
const auto rhs = std::bit_cast<int64_t>(b);
switch (opcode) {
case ValueOpcode::SLessThan64: return lhs < rhs;
case ValueOpcode::SLessThanEqual64: return lhs <= rhs;
case ValueOpcode::SGreaterThan64: return lhs > rhs;
default: return lhs >= rhs;
}
});
return;
case ValueOpcode::LogicalAnd:
if (!FoldLogical(inst, [](bool a, bool b) { return a && b; })) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, Type::U1)) {
Replace(inst, lhs.U1() ? rhs : lhs);
} else if (IsImmediate(rhs, Type::U1)) {
Replace(inst, rhs.U1() ? lhs : rhs);
}
}
return;
case ValueOpcode::LogicalOr:
if (!FoldLogical(inst, [](bool a, bool b) { return a || b; })) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, Type::U1)) {
Replace(inst, lhs.U1() ? lhs : rhs);
} else if (IsImmediate(rhs, Type::U1)) {
Replace(inst, rhs.U1() ? rhs : lhs);
}
}
return;
case ValueOpcode::LogicalXor:
if (!FoldLogical(inst, [](bool a, bool b) { return a != b; })) {
const auto lhs = Arg(inst, 0);
const auto rhs = Arg(inst, 1);
if (IsImmediate(lhs, Type::U1) && !lhs.U1()) {
Replace(inst, rhs);
} else if (IsImmediate(rhs, Type::U1) && !rhs.U1()) {
Replace(inst, lhs);
}
}
return;
case ValueOpcode::LogicalNot: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U1)) {
Replace(inst, Value(!value.U1()));
} else if (auto* producer = value.TryInstruction();
producer != nullptr && producer->GetOpcode() == ValueOpcode::LogicalNot) {
Replace(inst, producer->Arg(0));
}
return;
}
default: return;
}
}
} // namespace
void ConstantPropagationPass(const BlockList& blocks) {
for (auto* block: blocks) {
for (auto& inst: block->Instructions()) {
FoldInstruction(inst);
}
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,9 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void ConstantPropagationPass(const BlockList& blocks);
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,40 @@
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void RemoveIdentities(const BlockList& blocks) {
for (auto* block: blocks) {
auto& instructions = block->Instructions();
for (auto inst = instructions.begin(); inst != instructions.end();) {
if (inst->GetOpcode() != ValueOpcode::Identity) {
inst++;
continue;
}
const auto replacement = inst->Arg(0);
inst->ReplaceUsesWith(replacement, false);
inst = instructions.erase(inst);
}
}
}
void EliminateDeadCode(const BlockList& blocks) {
bool changed;
do {
changed = false;
for (auto block = blocks.rbegin(); block != blocks.rend(); block++) {
auto& instructions = (*block)->Instructions();
auto inst = instructions.end();
while (inst != instructions.begin()) {
--inst;
if (inst->HasUses() || inst->MayHaveSideEffects()) {
continue;
}
inst->Invalidate();
inst = instructions.erase(inst);
changed = true;
}
}
} while (changed);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,10 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void RemoveIdentities(const BlockList& blocks);
void EliminateDeadCode(const BlockList& blocks);
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,141 @@
#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
#include <algorithm>
#include <queue>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
struct ChainResult {
Value value;
Inst* write = nullptr;
};
ChainResult SearchChain(Value value, uint32_t lane, uint32_t wave_size) {
for (;;) {
value = value.Resolve();
auto* inst = value.TryInstruction();
if (inst == nullptr || inst->GetOpcode() != ValueOpcode::WriteLane) {
return {value};
}
const auto selector = inst->Arg(2).Resolve();
if (!selector.IsImmediate() || selector.GetType() != Type::U32) {
return {value};
}
if (selector.U32() % wave_size == lane) {
return {value, inst};
}
value = inst->Arg(0);
}
}
bool IsPossibleToEliminate(Value source, uint32_t lane, uint32_t wave_size) {
std::queue<Value> queue;
std::unordered_set<Inst*> visited;
queue.push(source);
while (!queue.empty()) {
const auto chain = SearchChain(queue.front(), lane, wave_size);
queue.pop();
if (chain.write != nullptr) {
continue;
}
auto* inst = chain.value.TryInstruction();
if (inst == nullptr || inst->GetOpcode() != ValueOpcode::Phi || inst->NumArgs() == 0) {
return false;
}
if (!visited.insert(inst).second) {
continue;
}
for (size_t index = inst->NumArgs(); index-- > 0;) {
queue.push(inst->Arg(index));
}
}
return true;
}
using PhiMap = std::unordered_map<Inst*, Inst*>;
Value GetRealValue(PhiMap& phi_map, Value source, uint32_t lane, uint32_t wave_size) {
const auto chain = SearchChain(source, lane, wave_size);
if (chain.write != nullptr) {
return chain.write->Arg(1);
}
auto* inst = chain.value.ResolveInstruction();
EXIT_IF(inst->GetOpcode() != ValueOpcode::Phi);
const auto [entry, is_new] = phi_map.try_emplace(inst);
if (!is_new) {
return Value(entry->second);
}
auto* block = inst->Parent();
auto insertion_point = std::find_if(block->begin(), block->end(),
[&](const Inst& candidate) { return &candidate == inst; });
EXIT_IF(insertion_point == block->end());
auto& phi = *block->PrependNewInst(insertion_point, ValueOpcode::Phi);
phi.SetFlags(Type::U32);
entry->second = &phi;
std::vector<Value> arguments;
arguments.reserve(inst->NumArgs());
for (size_t index = 0; index < inst->NumArgs(); index++) {
arguments.push_back(GetRealValue(phi_map, inst->Arg(index), lane, wave_size));
}
const auto first = arguments.front().Resolve();
if (std::ranges::all_of(arguments,
[&](Value argument) { return argument.Resolve() == first; })) {
phi.ReplaceUsesWith(first);
} else {
for (size_t index = 0; index < arguments.size(); index++) {
phi.AddPhiOperand(inst->PhiBlock(index), arguments[index]);
}
}
return Value(&phi);
}
} // namespace
ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size) {
ReadLaneStats stats;
if (wave_size != 32u && wave_size != 64u) {
return stats;
}
for (auto* block: program.blocks) {
for (auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::ReadLane) {
continue;
}
const auto selector = inst.Arg(1).Resolve();
if (!selector.IsImmediate() || selector.GetType() != Type::U32) {
continue;
}
const auto lane = selector.U32() % wave_size;
const auto chain = SearchChain(inst.Arg(0), lane, wave_size);
if (chain.write != nullptr) {
inst.ReplaceUsesWith(chain.write->Arg(1));
stats.rewritten_reads++;
continue;
}
auto* producer = chain.value.TryInstruction();
if (producer == nullptr || producer->GetOpcode() != ValueOpcode::Phi ||
!IsPossibleToEliminate(chain.value, lane, wave_size)) {
continue;
}
PhiMap phi_map;
inst.ReplaceUsesWith(GetRealValue(phi_map, chain.value, lane, wave_size));
stats.rewritten_reads++;
}
}
return stats;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,13 @@
#pragma once
#include "graphics/shader/recompiler/ir/ValueProgram.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct ReadLaneStats {
uint32_t rewritten_reads = 0;
};
[[nodiscard]] ReadLaneStats EliminateReadLane(ValueProgram& program, uint32_t wave_size);
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,492 @@
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include "graphics/shader/shaderBindings.h"
#include <algorithm>
#include <fmt/format.h>
#include <functional>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor,
Decoder::ImageDimension requested) {
const bool is_array = requested == Decoder::ImageDimension::Dim1DArray ||
requested == Decoder::ImageDimension::Dim2DArray ||
requested == Decoder::ImageDimension::Dim2DMsaaArray;
switch (static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu)) {
case Prospero::ImageType::kColor1D: return Decoder::ImageDimension::Dim1D;
case Prospero::ImageType::kColor1DArray:
if (is_array) {
return Decoder::ImageDimension::Dim1DArray;
}
return Decoder::ImageDimension::Dim1D;
case Prospero::ImageType::kColor3D: return Decoder::ImageDimension::Dim3D;
case Prospero::ImageType::kCube: return Decoder::ImageDimension::Dim2DArray;
case Prospero::ImageType::kColor2DArray:
if (is_array) {
return Decoder::ImageDimension::Dim2DArray;
}
return Decoder::ImageDimension::Dim2D;
case Prospero::ImageType::kColor2DMsaaArray:
if (is_array) {
return Decoder::ImageDimension::Dim2DMsaaArray;
}
return Decoder::ImageDimension::Dim2DMsaa;
case Prospero::ImageType::kColor2D: return Decoder::ImageDimension::Dim2D;
case Prospero::ImageType::kColor2DMsaa: return Decoder::ImageDimension::Dim2DMsaa;
default: return Decoder::ImageDimension::Unknown;
}
}
bool NullImageDescriptor(const DescriptorValue& descriptor) {
return descriptor.dwords[0] == 0 && (descriptor.dwords[1] & 0xffu) == 0;
}
bool ValidImageDescriptor(const DescriptorValue& descriptor) {
const auto type = static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu);
const auto format = static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
if (type < Prospero::ImageType::kColor1D || format == Prospero::BufferFormat::kInvalid) {
return false;
}
if (type == Prospero::ImageType::kColor2DMsaa ||
type == Prospero::ImageType::kColor2DMsaaArray) {
const auto base_level = (descriptor.dwords[3] >> 12u) & 0xfu;
const auto fragments = (descriptor.dwords[3] >> 16u) & 0xfu;
const auto max_mip = (descriptor.dwords[5] >> 4u) & 0xfu;
return base_level == 0 && fragments >= 1 && fragments <= 3 && max_mip == fragments;
}
return true;
}
uint32_t DescriptorImageSwizzle(const DescriptorValue& descriptor) {
return descriptor.dwords[3] & 0xfffu;
}
bool DescriptorIsCube(const DescriptorValue& descriptor) {
return static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu) ==
Prospero::ImageType::kCube;
}
bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResource& result) {
if (descriptor.dword_count != std::size(result.fields)) {
return false;
}
std::copy_n(descriptor.dwords.begin(), std::size(result.fields), result.fields);
return true;
}
uint64_t AddressSpecialization(const AddressResource& resource,
const ResourceSnapshot::Address& snapshot) {
return resource.kind == ResourceKind::Flat || resource.unbased
? snapshot.binding_base
: snapshot.guest_base - snapshot.binding_base;
}
bool IsImageOp(ValueOpcode op) {
switch (op) {
case ValueOpcode::ImageQueryDimensions:
case ValueOpcode::ImageQueryLod:
case ValueOpcode::ImageRead:
case ValueOpcode::ImageWrite:
case ValueOpcode::ImageSampleRaw:
case ValueOpcode::ImageGatherRaw:
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
case ValueOpcode::ImageAtomicAnd32:
case ValueOpcode::ImageAtomicOr32:
case ValueOpcode::ImageAtomicXor32: return true;
default: return false;
}
}
} // namespace
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot,
std::string* error) {
if (!program.resource_tracking_complete) {
if (error != nullptr) {
*error = "shader resources were not tracked";
}
return false;
}
if (snapshot.buffers.size() != program.info.buffers.size() ||
snapshot.images.size() != program.info.images.size() ||
snapshot.samplers.size() != program.info.samplers.size() ||
snapshot.addresses.size() != program.info.addresses.size()) {
if (error != nullptr) {
*error = "resource snapshot does not match dense shader topology";
}
return false;
}
if (program.values == nullptr ||
snapshot.flattened_srt.size() != program.values->srt_reads.size()) {
if (error != nullptr) {
*error = "flattened SRT snapshot does not match the shader plan";
}
return false;
}
if (program.binding_layout_complete) {
for (const auto reg: program.bindings.user_data_registers) {
if (reg < program.user_data_base ||
reg - program.user_data_base >= snapshot.user_data.size()) {
if (error != nullptr) {
*error = fmt::format("runtime snapshot is missing user SGPR {}", reg);
}
return false;
}
}
}
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto alias = program.info.buffers[i].image_alias;
if (alias != BufferResource::NoImageAlias && alias >= program.info.images.size()) {
if (error != nullptr) {
*error = fmt::format("buffer resource {} has invalid image alias {}", i, alias);
}
return false;
}
}
const auto CheckWidth = [&](const auto& values, uint32_t width, const char* kind) {
for (uint32_t i = 0; i < values.size(); i++) {
if (values[i].dword_count != width) {
if (error != nullptr) {
*error = fmt::format("{} descriptor {} has {} dwords", kind, i,
values[i].dword_count);
}
return false;
}
}
return true;
};
for (uint32_t i = 0; i < snapshot.addresses.size(); i++) {
if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) {
if (error != nullptr) {
*error = fmt::format("address resource {} binds above its guest base", i);
}
return false;
}
}
return CheckWidth(snapshot.buffers, 4, "buffer") && CheckWidth(snapshot.images, 8, "image") &&
CheckWidth(snapshot.samplers, 4, "sampler");
}
bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot,
std::string* error) {
if (!ValidateResourceSnapshot(program, snapshot, error)) {
return false;
}
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto& buffer = program.info.buffers[i];
ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(snapshot.buffers[i], descriptor)) {
if (error != nullptr) {
*error = fmt::format("buffer descriptor {} has invalid width", i);
}
return false;
}
if (buffer.packed_stride != descriptor.PackedStride() ||
buffer.descriptor_format != descriptor.Format()) {
if (error != nullptr) {
*error = fmt::format("buffer descriptor {} no longer matches specialization", i);
}
return false;
}
}
for (uint32_t i = 0; i < program.info.images.size(); i++) {
const auto& image = program.info.images[i];
const auto& descriptor = snapshot.images[i];
if (NullImageDescriptor(descriptor)) {
bool canonical_kind =
image.kind == ResourceKind::Image || image.kind == ResourceKind::StorageImage;
if (image.atomic) {
canonical_kind = image.kind == ResourceKind::StorageImageUint;
}
if (image.dimension != Decoder::ImageDimension::Dim2D || image.cube ||
!canonical_kind) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} no longer matches canonical null specialization", i);
}
return false;
}
continue;
}
const auto dimension = DescriptorDimension(descriptor, image.dimension);
if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension ||
DescriptorIsCube(descriptor) != image.cube) {
if (error != nullptr) {
*error =
fmt::format("image descriptor {} no longer matches specialized dimension: "
"{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
descriptor.dwords[6], descriptor.dwords[7]);
}
return false;
}
if (image.kind == ResourceKind::Image || image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint) {
const bool storage = image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint;
if (storage && image.storage_swizzle != DescriptorImageSwizzle(descriptor)) {
if (error != nullptr) {
*error = fmt::format("storage image descriptor {} changed swizzle", i);
}
return false;
}
const auto format =
static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
const bool raw_sint_storage = storage && format == Prospero::BufferFormat::k32SInt &&
!image.read && !image.atomic;
const bool uint_descriptor = Prospero::IsUintTextureFormat(format) || raw_sint_storage;
const auto uint_program = image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImageUint;
if (uint_descriptor != uint_program && !(image.atomic && uint_program)) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} no longer matches specialized format: "
"{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x},{:08x}",
i, descriptor.dwords[0], descriptor.dwords[1], descriptor.dwords[2],
descriptor.dwords[3], descriptor.dwords[4], descriptor.dwords[5],
descriptor.dwords[6], descriptor.dwords[7]);
}
return false;
}
if (image.depth_compare && uint_program) {
if (error != nullptr) {
*error = fmt::format("integer image descriptor {} uses depth comparison", i);
}
return false;
}
}
}
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
if (program.info.addresses[i].specialized_base !=
AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) {
if (error != nullptr) {
*error = fmt::format("address resource {} no longer matches specialization", i);
}
return false;
}
}
return true;
}
bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot& snapshot, std::string* error) {
if (!program.resource_tracking_complete) {
if (error != nullptr) {
*error = "shader resources were not tracked";
}
return false;
}
std::vector<DescriptorSourceRequest> requests;
requests.reserve(program.info.buffers.size() + program.info.images.size() +
program.info.samplers.size() + program.info.addresses.size());
for (const auto& buffer: program.info.buffers) {
requests.push_back({buffer.source, buffer.first_use_pc});
}
for (const auto& image: program.info.images) {
requests.push_back({image.source, image.first_use_pc});
}
for (const auto& sampler: program.info.samplers) {
requests.push_back({sampler.source, sampler.first_use_pc});
}
for (const auto& address: program.info.addresses) {
if (!address.unbased) {
requests.push_back({address.source, address.first_use_pc});
}
}
std::vector<DescriptorValue> values;
std::vector<uint32_t> flattened_srt;
if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, error)) {
return false;
}
ResourceSnapshot next;
auto cursor = values.begin();
next.buffers.assign(cursor, cursor + program.info.buffers.size());
cursor += program.info.buffers.size();
for (auto& descriptor: next.buffers) {
ShaderBufferResource buffer;
if (DecodeBufferDescriptor(descriptor, buffer) && buffer.Type() != 0) {
descriptor.dwords.fill(0);
}
}
next.images.assign(cursor, cursor + program.info.images.size());
cursor += program.info.images.size();
for (auto& descriptor: next.images) {
if (!ValidImageDescriptor(descriptor)) {
descriptor.dwords.fill(0);
}
}
next.samplers.assign(cursor, cursor + program.info.samplers.size());
cursor += program.info.samplers.size();
for (const auto& address: program.info.addresses) {
if (!address.unbased) {
const auto value = *cursor++;
auto base = (static_cast<uint64_t>(value.dwords[0]) |
static_cast<uint64_t>(value.dwords[1]) << 32u) &
AddressMask;
if (address.kind == ResourceKind::ScalarBuffer) {
base &= ~uint64_t {3};
}
const auto before = static_cast<uint64_t>(-static_cast<int64_t>(address.min_offset));
uint64_t binding_base = 0;
if (address.kind == ResourceKind::Flat) {
binding_base = base & ~(FlatAddressWindowSize - 1u);
} else if (base >= before) {
binding_base = base - before;
}
next.addresses.push_back({base, binding_base});
} else {
if (!runtime.flat_memory_base.has_value()) {
if (error != nullptr) {
*error =
fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
"guest-address translation",
address.kind == ResourceKind::Flat ? "FLAT" : "global/scratch",
address.first_use_pc);
}
return false;
}
next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base});
}
}
next.flattened_srt = std::move(flattened_srt);
next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
if (!ValidateResourceSnapshot(program, next, error)) {
return false;
}
snapshot = std::move(next);
return true;
}
bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std::string* error) {
if (!program.resource_tracking_complete || program.shader_info_complete ||
program.binding_layout_complete) {
if (error != nullptr) {
*error = !program.resource_tracking_complete ? "shader resources were not tracked"
: "resource specialization is too late";
}
return false;
}
if (!ValidateResourceSnapshot(program, snapshot, error)) {
return false;
}
auto next = program.info;
for (uint32_t i = 0; i < next.buffers.size(); i++) {
ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(snapshot.buffers[i], descriptor)) {
if (error != nullptr) {
*error = fmt::format("buffer descriptor {} has invalid width", i);
}
return false;
}
next.buffers[i].packed_stride = descriptor.PackedStride();
next.buffers[i].descriptor_format = descriptor.Format();
}
for (uint32_t i = 0; i < next.addresses.size(); i++) {
next.addresses[i].specialized_base =
AddressSpecialization(next.addresses[i], snapshot.addresses[i]);
}
for (uint32_t i = 0; i < next.images.size(); i++) {
const auto& descriptor = snapshot.images[i];
auto& image = next.images[i];
if (NullImageDescriptor(descriptor)) {
image.dimension = Decoder::ImageDimension::Dim2D;
image.cube = false;
switch (image.kind) {
case ResourceKind::ImageUint: image.kind = ResourceKind::Image; break;
case ResourceKind::StorageImageUint:
if (!image.atomic) {
image.kind = ResourceKind::StorageImage;
}
break;
default: break;
}
continue;
}
const auto descriptor_dimension = DescriptorDimension(descriptor, image.dimension);
if (descriptor_dimension == Decoder::ImageDimension::Unknown) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} has unsupported type {}: {:08x},{:08x},{:08x},{:08x},"
"{:08x},{:08x},{:08x},{:08x}",
i, (descriptor.dwords[3] >> 28u) & 0xfu, descriptor.dwords[0],
descriptor.dwords[1], descriptor.dwords[2], descriptor.dwords[3],
descriptor.dwords[4], descriptor.dwords[5], descriptor.dwords[6],
descriptor.dwords[7]);
}
return false;
}
image.dimension = descriptor_dimension;
image.cube = DescriptorIsCube(descriptor);
if (image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint) {
image.storage_swizzle = DescriptorImageSwizzle(descriptor);
}
const auto format =
static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
const bool storage = image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint;
const bool raw_sint_storage =
storage && format == Prospero::BufferFormat::k32SInt && !image.read && !image.atomic;
const bool uint_image = Prospero::IsUintTextureFormat(format) || raw_sint_storage;
if (uint_image) {
switch (image.kind) {
case ResourceKind::Image: image.kind = ResourceKind::ImageUint; break;
case ResourceKind::StorageImage: image.kind = ResourceKind::StorageImageUint; break;
default: break;
}
}
}
if (program.values == nullptr) {
if (error != nullptr) {
*error = "typed SSA is not ready";
}
return false;
}
auto memory_info = program.values->memory_info;
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (!IsImageOp(inst.GetOpcode())) {
continue;
}
const auto index = inst.Flags<MemoryFlags>().index;
if (index >= memory_info.size()) {
if (error != nullptr) {
*error = fmt::format("typed image instruction has invalid memory metadata {}",
index);
}
return false;
}
auto& memory = memory_info[index];
if (memory.resource >= next.images.size()) {
if (error != nullptr) {
*error = fmt::format("typed image instruction has invalid resource {}",
memory.resource);
}
return false;
}
const auto& image = next.images[memory.resource];
memory.kind = image.kind;
memory.image_dimension = image.dimension;
memory.image_cube = image.cube;
}
}
program.info = std::move(next);
program.values->memory_info = std::move(memory_info);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,42 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
constexpr uint64_t FlatAddressWindowSize = 0x10000;
struct ResourceSnapshot {
struct Address {
uint64_t guest_base = 0;
uint64_t binding_base = 0;
bool operator==(const Address& other) const = default;
};
std::vector<DescriptorValue> buffers;
std::vector<DescriptorValue> images;
std::vector<DescriptorValue> samplers;
std::vector<Address> addresses;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
};
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot,
std::string* error);
bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot,
std::string* error);
// Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot.
// On failure the destination is unchanged.
bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot& snapshot, std::string* error);
// Applies runtime descriptor shape/format facts to a copied dense topology before layout and
// emission. On failure the program is unchanged.
bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */
@@ -0,0 +1,677 @@
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include <algorithm>
#include <fmt/format.h>
#include <unordered_set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint32_t SamplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u);
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
bool IsBufferAtomic(ValueOpcode op) {
switch (op) {
case ValueOpcode::BufferAtomicSwap32:
case ValueOpcode::BufferAtomicIAdd32:
case ValueOpcode::BufferAtomicISub32:
case ValueOpcode::BufferAtomicSMin32:
case ValueOpcode::BufferAtomicUMin32:
case ValueOpcode::BufferAtomicSMax32:
case ValueOpcode::BufferAtomicUMax32:
case ValueOpcode::BufferAtomicAnd32:
case ValueOpcode::BufferAtomicOr32:
case ValueOpcode::BufferAtomicXor32:
case ValueOpcode::BufferAtomicFMin32:
case ValueOpcode::BufferAtomicFMax32: return true;
default: return false;
}
}
bool IsBufferStore(ValueOpcode op) {
switch (op) {
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return true;
default: return IsBufferAtomic(op);
}
}
bool IsBuffer(ValueOpcode op) {
switch (op) {
case ValueOpcode::ReadConstBuffer:
case ValueOpcode::LoadBufferU8:
case ValueOpcode::LoadBufferU16:
case ValueOpcode::LoadBufferU32:
case ValueOpcode::StoreBufferU8:
case ValueOpcode::StoreBufferU16:
case ValueOpcode::StoreBufferU32: return true;
default: return IsBufferAtomic(op);
}
}
bool IsAddressStore(ValueOpcode op) {
switch (op) {
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32: return true;
default: return false;
}
}
bool IsAddress(ValueOpcode op) {
switch (op) {
case ValueOpcode::LoadAddressU8:
case ValueOpcode::LoadAddressU16:
case ValueOpcode::LoadAddressU32:
case ValueOpcode::StoreAddressU8:
case ValueOpcode::StoreAddressU16:
case ValueOpcode::StoreAddressU32: return true;
default: return false;
}
}
bool IsImageAtomic(ValueOpcode op) {
switch (op) {
case ValueOpcode::ImageAtomicIAdd32:
case ValueOpcode::ImageAtomicUMin32:
case ValueOpcode::ImageAtomicUMax32:
case ValueOpcode::ImageAtomicAnd32:
case ValueOpcode::ImageAtomicOr32:
case ValueOpcode::ImageAtomicXor32: return true;
default: return false;
}
}
bool IsImageStore(ValueOpcode op) {
return op == ValueOpcode::ImageWrite || IsImageAtomic(op);
}
bool IsImage(ValueOpcode op) {
switch (op) {
case ValueOpcode::ImageQueryDimensions:
case ValueOpcode::ImageQueryLod:
case ValueOpcode::ImageRead:
case ValueOpcode::ImageWrite:
case ValueOpcode::ImageSampleRaw:
case ValueOpcode::ImageGatherRaw: return true;
default: return IsImageAtomic(op);
}
}
bool NeedsSampler(ValueOpcode op) {
return op == ValueOpcode::ImageQueryLod || op == ValueOpcode::ImageSampleRaw ||
op == ValueOpcode::ImageGatherRaw;
}
uint32_t ByteExtent(const MemoryInfo& memory) {
const auto bytes = std::max((memory.data_bits + 7u) / 8u, 1u);
const auto count = std::max(memory.data_dwords, 1u);
const auto end = static_cast<uint64_t>(memory.offset) + static_cast<uint64_t>(bytes) * count;
return end > UINT32_MAX ? UINT32_MAX : static_cast<uint32_t>(end);
}
ImageMipMode MipMode(const MemoryInfo& memory) {
const bool storage =
memory.kind == ResourceKind::StorageImage || memory.kind == ResourceKind::StorageImageUint;
return storage && memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
}
class Tracker {
public:
Tracker(Program& program, ValueProgram& values)
: m_program(program), m_values(values), m_info(program.info) {
m_info.buffers.clear();
m_info.addresses.clear();
m_info.images.clear();
m_info.samplers.clear();
m_info.sampled_pairs.clear();
}
bool Run(std::string* error) {
if (m_program.resource_tracking_complete) {
return Fail(0, error, "resources already tracked");
}
if (!m_program.srt_plan_complete) {
return Fail(0, error, "SRT plan is not ready");
}
for (auto* block: m_values.blocks) {
for (auto& inst: *block) {
if (!Collect(inst, error)) {
return false;
}
}
}
LinkImageAliases();
for (const auto& patch: m_handle_patches) {
patch.handle->SetFlags<uint32_t>(patch.resource);
}
for (const auto& patch: m_memory_patches) {
auto& memory = m_values.memory_info[patch.index];
memory.resource = patch.resource;
if (patch.has_sampler) {
memory.sampler = patch.sampler;
}
}
m_values.descriptor_sources = std::move(m_sources);
m_program.info = std::move(m_info);
m_program.resource_tracking_complete = true;
return true;
}
private:
struct HandlePatch {
Inst* handle = nullptr;
uint32_t resource = 0;
};
struct MemoryPatch {
uint32_t index = 0;
uint32_t resource = 0;
uint32_t sampler = 0;
bool has_sampler = false;
};
bool Fail(uint32_t pc, std::string* error, const std::string& reason) const {
if (error != nullptr) {
*error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}",
m_program.shader_hash, StageName(m_program.stage), pc, reason);
}
return false;
}
struct AddressPart {
Value value;
bool rooted = false;
};
AddressPart FindAddressPart(Value value, std::unordered_set<const Inst*>& visiting) const {
value = value.Resolve();
if (value.IsImmediate()) {
return {value, false};
}
const auto* inst = value.TryInstruction();
if (inst == nullptr || !visiting.insert(inst).second) {
return {};
}
const auto finish = [&](AddressPart part) {
visiting.erase(inst);
return part;
};
std::string reason;
if (ValidateRuntimeValue(m_program, value, reason)) {
return finish({value, true});
}
const auto merge = [&](AddressPart left, AddressPart right, bool require_both) {
if (left.value.IsEmpty() || right.value.IsEmpty()) {
return require_both ? AddressPart {} : (left.value.IsEmpty() ? right : left);
}
return EquivalentValue(m_values, left.value, right.value)
? AddressPart {left.value, left.rooted || right.rooted}
: AddressPart {};
};
switch (inst->GetOpcode()) {
case ValueOpcode::SelectU32:
return finish(merge(FindAddressPart(inst->Arg(1), visiting),
FindAddressPart(inst->Arg(2), visiting), true));
case ValueOpcode::Phi: {
const auto invariant = ResolveInvariantPhi(m_values, value);
return finish(invariant.IsEmpty() ? AddressPart {}
: FindAddressPart(invariant, visiting));
}
case ValueOpcode::IAdd32:
case ValueOpcode::ISub32: {
const auto left = FindAddressPart(inst->Arg(0), visiting);
const auto right = FindAddressPart(inst->Arg(1), visiting);
if (left.rooted == right.rooted ||
(inst->GetOpcode() == ValueOpcode::ISub32 && right.rooted)) {
return finish({});
}
return finish(left.rooted ? left : right);
}
case ValueOpcode::CompositeExtractU32x2: {
const auto* source = inst->Arg(0).ResolveInstruction();
if (source == nullptr || source->GetOpcode() != ValueOpcode::IAddCarry32) {
return finish({});
}
const auto left = FindAddressPart(source->Arg(0), visiting);
const auto right = FindAddressPart(source->Arg(1), visiting);
if (left.rooted == right.rooted) {
return finish({});
}
return finish(left.rooted ? left : right);
}
default: return finish({});
}
}
bool MakeFlatAddressSource(const Inst& handle, DescriptorSource& descriptor) const {
std::unordered_set<const Inst*> visiting;
auto low = FindAddressPart(handle.Arg(0), visiting);
visiting.clear();
auto high = FindAddressPart(handle.Arg(1), visiting);
if ((!low.rooted && !high.rooted) || low.value.IsEmpty() || high.value.IsEmpty()) {
return false;
}
descriptor.dword_count = 2;
descriptor.dwords[0] = low.value;
descriptor.dwords[1] = high.value;
return true;
}
bool MakeSource(const Inst& handle, uint32_t width, bool sampler, DescriptorSource& descriptor,
uint32_t pc, std::string* error) const {
if (handle.NumArgs() != width) {
return Fail(pc, error,
fmt::format("{} has {} descriptor dwords, expected {}",
ValueOpcodeName(handle.GetOpcode()), handle.NumArgs(), width));
}
descriptor.dword_count = width;
for (uint32_t i = 0; i < width; i++) {
descriptor.dwords[i] = handle.Arg(i).Resolve();
}
const auto dword0 = descriptor.dwords[0].Resolve();
if (sampler && dword0.IsImmediate() && dword0.GetType() == Type::U32 &&
(dword0.U32() & SamplerBorderClampMask) == 0) {
// Border color and its table index are unused unless a clamp axis selects border mode.
descriptor.dwords[3] = Value(0u);
}
return true;
}
bool ValidateSource(const DescriptorSource& descriptor, std::string& reason,
uint32_t& bad_dword) const {
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
bad_dword = i;
if (descriptor.dwords[i].Resolve().GetType() != Type::U32) {
reason = "is not U32";
return false;
}
if (!ValidateRuntimeValue(m_program, descriptor.dwords[i], reason)) {
return false;
}
}
return true;
}
uint32_t InternSource(const DescriptorSource& descriptor) {
for (uint32_t candidate = 0; candidate < m_sources.size(); candidate++) {
const auto& current = m_sources[candidate];
if (current.dword_count != descriptor.dword_count) {
continue;
}
bool same = true;
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
same = same && EquivalentValue(m_values, current.dwords[i], descriptor.dwords[i]);
}
if (same) {
return candidate;
}
}
m_sources.push_back(descriptor);
return static_cast<uint32_t>(m_sources.size() - 1);
}
bool GetHandle(Value value, ValueOpcode expected, uint32_t width, uint32_t pc, Inst*& handle,
uint32_t& source, std::string* error, bool sampler = false) {
handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != expected) {
return Fail(pc, error,
fmt::format("memory operation requires {}", ValueOpcodeName(expected)));
}
DescriptorSource descriptor;
if (!MakeSource(*handle, width, sampler, descriptor, pc, error)) {
return false;
}
std::string reason;
uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) {
return Fail(
pc, error,
fmt::format("{} dword {} {}", ValueOpcodeName(expected), bad_dword, reason));
}
source = InternSource(descriptor);
return true;
}
bool GetAddressHandle(const Inst& memory_inst, Value value, uint32_t pc, Inst*& handle,
uint32_t& source, bool& unbased, std::string* error) {
handle = value.Resolve().TryInstruction();
if (handle == nullptr || handle->GetOpcode() != ValueOpcode::GetAddressResource) {
return Fail(pc, error, "address operation requires GetAddressResource");
}
if (handle->NumArgs() != 2) {
return Fail(pc, error, "GetAddressResource must have two address dwords");
}
DescriptorSource descriptor;
const auto& memory = m_values.memory_info[memory_inst.Flags<MemoryFlags>().index];
if (memory.address_is_full) {
if (memory.kind != ResourceKind::Flat || !MakeFlatAddressSource(*handle, descriptor)) {
unbased = true;
source = UINT32_MAX;
return true;
}
} else if (!MakeSource(*handle, 2, false, descriptor, pc, error)) {
return false;
}
std::string reason;
uint32_t bad_dword = 0;
if (!ValidateSource(descriptor, reason, bad_dword)) {
if (memory.address_is_full) {
unbased = true;
source = UINT32_MAX;
return true;
}
return Fail(
pc, error,
fmt::format("scalar memory base dword {} is unresolved: {}", bad_dword, reason));
}
unbased = false;
source = InternSource(descriptor);
return true;
}
uint32_t AddBuffer(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
for (uint32_t i = 0; i < m_info.buffers.size(); i++) {
if (m_info.buffers[i].source == source) {
Merge(m_info.buffers[i], memory, op, pc);
return i;
}
}
if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) {
return UINT32_MAX;
}
BufferResource resource;
resource.source = source;
resource.first_use_pc = pc;
Merge(resource, memory, op, pc);
m_info.buffers.push_back(resource);
return static_cast<uint32_t>(m_info.buffers.size() - 1);
}
static void Merge(BufferResource& resource, const MemoryInfo& memory, ValueOpcode op,
uint32_t pc) {
const bool atomic = IsBufferAtomic(op);
const bool write = IsBufferStore(op);
resource.first_use_pc = std::min(resource.first_use_pc, pc);
resource.max_byte_extent = std::max(resource.max_byte_extent, ByteExtent(memory));
resource.read = resource.read || !write || atomic;
resource.written = resource.written || write;
resource.atomic = resource.atomic || atomic;
resource.formatted = resource.formatted || memory.formatted;
resource.scalar = resource.scalar || op == ValueOpcode::ReadConstBuffer ||
memory.kind == ResourceKind::ScalarBuffer;
}
uint32_t AddImage(uint32_t source, const MemoryInfo& memory, ValueOpcode op, uint32_t pc) {
const auto mip = MipMode(memory);
const bool depth = (memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0;
for (uint32_t i = 0; i < m_info.images.size(); i++) {
auto& image = m_info.images[i];
if (image.source == source && image.kind == memory.kind &&
image.dimension == memory.image_dimension && image.mip_mode == mip &&
image.depth_compare == depth) {
Merge(image, op, pc);
return i;
}
}
if (m_info.images.size() >= ShaderInfo::MaxImages) {
return UINT32_MAX;
}
ImageResource image;
image.source = source;
image.first_use_pc = pc;
image.kind = memory.kind;
image.dimension = memory.image_dimension;
image.mip_mode = mip;
image.depth_compare = depth;
Merge(image, op, pc);
m_info.images.push_back(image);
return static_cast<uint32_t>(m_info.images.size() - 1);
}
static void Merge(ImageResource& image, ValueOpcode op, uint32_t pc) {
const bool atomic = IsImageAtomic(op);
const bool write = IsImageStore(op);
image.first_use_pc = std::min(image.first_use_pc, pc);
image.read = image.read || !write || atomic;
image.written = image.written || write;
image.atomic = image.atomic || atomic;
}
uint32_t AddSampler(uint32_t source, uint32_t pc) {
for (uint32_t i = 0; i < m_info.samplers.size(); i++) {
if (m_info.samplers[i].source == source) {
m_info.samplers[i].first_use_pc = std::min(m_info.samplers[i].first_use_pc, pc);
return i;
}
}
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
return UINT32_MAX;
}
m_info.samplers.push_back({source, pc});
return static_cast<uint32_t>(m_info.samplers.size() - 1);
}
uint32_t AddAddress(uint32_t source, bool unbased, const MemoryInfo& memory, ValueOpcode op,
uint32_t pc) {
auto immediate = static_cast<int32_t>(memory.offset);
if (memory.kind == ResourceKind::ScalarBuffer) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto min_offset = unbased ? 0 : std::min(immediate, 0);
for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
auto& address = m_info.addresses[i];
if (address.source == source && address.unbased == unbased &&
address.kind == memory.kind) {
address.first_use_pc = std::min(address.first_use_pc, pc);
address.min_offset = std::min(address.min_offset, min_offset);
address.read = address.read || !IsAddressStore(op);
address.written = address.written || IsAddressStore(op);
return i;
}
}
if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
return UINT32_MAX;
}
AddressResource address;
address.source = source;
address.first_use_pc = pc;
address.kind = memory.kind;
address.min_offset = min_offset;
address.unbased = unbased;
address.read = !IsAddressStore(op);
address.written = IsAddressStore(op);
m_info.addresses.push_back(address);
return static_cast<uint32_t>(m_info.addresses.size() - 1);
}
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) {
pair.first_use_pc = std::min(pair.first_use_pc, pc);
return true;
}
}
if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) {
return Fail(pc, error, "sampled image/sampler pair limit exceeded");
}
m_info.sampled_pairs.push_back({image, sampler, pc});
return true;
}
bool AddHandlePatch(Inst* handle, uint32_t resource, uint32_t pc, std::string* error) {
for (const auto& patch: m_handle_patches) {
if (patch.handle == handle) {
return patch.resource == resource ||
Fail(pc, error,
fmt::format("{} is reused with incompatible resource classes",
ValueOpcodeName(handle->GetOpcode())));
}
}
m_handle_patches.push_back({handle, resource});
return true;
}
bool AddMemoryPatch(uint32_t index, uint32_t resource, uint32_t sampler, bool has_sampler,
uint32_t pc, std::string* error) {
for (auto& patch: m_memory_patches) {
if (patch.index != index) {
continue;
}
if (patch.resource != resource ||
(has_sampler && patch.has_sampler && patch.sampler != sampler)) {
return Fail(pc, error, "memory metadata is reused with incompatible resources");
}
if (has_sampler) {
patch.sampler = sampler;
patch.has_sampler = true;
}
return true;
}
m_memory_patches.push_back({index, resource, sampler, has_sampler});
return true;
}
bool Collect(Inst& inst, std::string* error) {
const auto op = inst.GetOpcode();
if (!IsBuffer(op) && !IsAddress(op) && !IsImage(op)) {
return true;
}
const auto flags = inst.Flags<MemoryFlags>();
if (flags.index >= m_values.memory_info.size()) {
return Fail(flags.pc, error,
fmt::format("memory metadata index {} is out of range", flags.index));
}
if (inst.NumArgs() == 0) {
return Fail(flags.pc, error, "memory operation has no resource handle");
}
const auto& memory = m_values.memory_info[flags.index];
if (memory.planning_only) {
return true;
}
Inst* handle = nullptr;
uint32_t source = 0;
uint32_t resource = 0;
if (IsBuffer(op)) {
if (!GetHandle(inst.Arg(0), ValueOpcode::GetBufferResource, 4, flags.pc, handle, source,
error)) {
return false;
}
resource = AddBuffer(source, memory, op, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "buffer resource limit exceeded");
}
return AddHandlePatch(handle, resource, flags.pc, error) &&
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
}
if (IsAddress(op)) {
bool unbased = false;
if (!GetAddressHandle(inst, inst.Arg(0), flags.pc, handle, source, unbased, error)) {
return false;
}
resource = AddAddress(source, unbased, memory, op, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "address resource limit exceeded");
}
return AddHandlePatch(handle, resource, flags.pc, error) &&
AddMemoryPatch(flags.index, resource, 0, false, flags.pc, error);
}
if (!GetHandle(inst.Arg(0), ValueOpcode::GetImageResource, 8, flags.pc, handle, source,
error)) {
return false;
}
resource = AddImage(source, memory, op, flags.pc);
if (resource == UINT32_MAX) {
return Fail(flags.pc, error, "image resource limit exceeded");
}
if (!AddHandlePatch(handle, resource, flags.pc, error)) {
return false;
}
uint32_t sampler = 0;
if (NeedsSampler(op)) {
if (inst.NumArgs() < 2) {
return Fail(flags.pc, error, "sampled image operation has no sampler handle");
}
Inst* sampler_handle = nullptr;
uint32_t sampler_source = 0;
if (!GetHandle(inst.Arg(1), ValueOpcode::GetSamplerResource, 4, flags.pc,
sampler_handle, sampler_source, error, true)) {
return false;
}
sampler = AddSampler(sampler_source, flags.pc);
if (sampler == UINT32_MAX) {
return Fail(flags.pc, error, "sampler resource limit exceeded");
}
if (!AddHandlePatch(sampler_handle, sampler, flags.pc, error) ||
!AddSampledPair(resource, sampler, flags.pc, error)) {
return false;
}
}
return AddMemoryPatch(flags.index, resource, sampler, NeedsSampler(op), flags.pc, error);
}
const DescriptorSource* Source(uint32_t source) const {
return source < m_sources.size() ? &m_sources[source] : nullptr;
}
void LinkImageAliases() {
for (auto& buffer: m_info.buffers) {
const auto* buffer_source = Source(buffer.source);
if (buffer_source == nullptr || buffer_source->dword_count != 4) {
continue;
}
for (uint32_t image = 0; image < m_info.images.size(); image++) {
const auto* image_source = Source(m_info.images[image].source);
if (image_source == nullptr || image_source->dword_count != 8) {
continue;
}
bool alias = true;
for (uint32_t dword = 0; dword < 4; dword++) {
alias = alias && EquivalentValue(m_values, buffer_source->dwords[dword],
image_source->dwords[dword]);
}
if (alias) {
buffer.image_alias = image;
break;
}
}
}
}
Program& m_program;
ValueProgram& m_values;
ShaderInfo m_info;
std::vector<DescriptorSource> m_sources;
std::vector<HandlePatch> m_handle_patches;
std::vector<MemoryPatch> m_memory_patches;
};
} // namespace
bool TrackResources(Program& program, std::string* error) {
if (program.values == nullptr) {
if (error != nullptr) {
*error = "shader resource tracking requires typed IR";
}
return false;
}
return Tracker(program, *program.values).Run(error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,14 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
// descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
bool TrackResources(Program& program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_ */
@@ -0,0 +1,318 @@
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components,
std::string name) {
const auto input = std::find_if(info.inputs.begin(), info.inputs.end(), [=](const auto& value) {
return value.kind == kind && value.location == location;
});
if (input == info.inputs.end()) {
info.inputs.push_back({kind, location, components, std::move(name)});
} else {
input->component_count = std::max(input->component_count, components);
}
}
bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) {
return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) {
return output.kind == kind && output.index == index;
});
}
void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t location,
std::string name) {
if (!HasOutput(info, kind, index)) {
info.outputs.push_back({kind, index, location, std::move(name)});
}
}
bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) {
auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
switch (program.stage) {
case ShaderType::Vertex:
if (options.vertex == nullptr) {
return Fail("vertex shader info requires vertex metadata");
}
if (options.vertex->resources_num < 0 ||
options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) {
return Fail("vertex resource count is out of range");
}
return true;
case ShaderType::Pixel:
if (options.pixel == nullptr) {
return Fail("pixel shader info requires pixel metadata");
}
return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) ||
Fail("pixel input count is out of range");
case ShaderType::Compute:
if (options.compute == nullptr) {
return Fail("compute shader info requires compute metadata");
}
return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) ||
Fail("compute thread ID count is out of range");
default: return Fail("unsupported shader stage for info collection");
}
}
bool ValidateValueReferences(const Program& program, const ShaderInfoOptions& options,
std::string* error) {
const auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
switch (inst.GetOpcode()) {
case ValueOpcode::GetAttribute: {
if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 ||
!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) {
return Fail("typed attribute reference is not constant");
}
if (program.stage == ShaderType::Vertex &&
(inst.Arg(1).U32() >= 4u || options.vertex == nullptr ||
inst.Arg(0).U32() >=
static_cast<uint32_t>(options.vertex->resources_num))) {
return Fail("vertex input reference is out of range");
}
break;
}
case ValueOpcode::GetBuiltin: {
if (!inst.Arg(0).IsImmediate() || inst.Arg(0).GetType() != Type::U32 ||
!inst.Arg(1).IsImmediate() || inst.Arg(1).GetType() != Type::U32) {
return Fail("typed builtin reference is not constant");
}
const auto kind = static_cast<StageInputKind>(inst.Arg(0).U32());
const auto component = inst.Arg(1).U32();
switch (kind) {
case StageInputKind::VertexIndex:
case StageInputKind::InstanceIndex:
case StageInputKind::FrontFacing:
case StageInputKind::LocalInvocationIndex:
if (component != 0u) {
return Fail("typed scalar builtin component is out of range");
}
break;
case StageInputKind::FragCoord:
if (component >= 4u) {
return Fail("typed fragment-coordinate component is out of range");
}
break;
case StageInputKind::WorkgroupId:
case StageInputKind::LocalInvocationId:
case StageInputKind::GlobalInvocationId:
if (component >= 3u) {
return Fail("typed invocation builtin component is out of range");
}
break;
case StageInputKind::Parameter:
default: return Fail("typed builtin kind is invalid");
}
break;
}
case ValueOpcode::SetAttribute:
if (inst.Flags<ExportFlags>().index >= program.values->export_info.size()) {
return Fail("typed export metadata index is out of range");
}
break;
default: break;
}
}
}
return true;
}
void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex,
ShaderInfo& info) {
AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex");
AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex");
if (vertex == nullptr) {
return;
}
uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {};
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() == ValueOpcode::GetAttribute) {
const auto attr = inst.Arg(0).U32();
const auto chan = inst.Arg(1).U32();
used_components[attr] = std::max(used_components[attr], chan + 1u);
}
}
}
for (uint32_t attr = 0; attr < static_cast<uint32_t>(vertex->resources_num) &&
attr < ShaderVertexInputInfo::RES_MAX;
attr++) {
if (used_components[attr] != 0) {
AddInput(info, StageInputKind::Parameter, attr, used_components[attr],
fmt::format("in_attr_{}", attr));
}
}
}
void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
if (pixel == nullptr) {
return;
}
if (pixel->HasPositionInput()) {
AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord");
}
if (pixel->ps_front_face) {
AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing");
}
for (uint32_t input = 0; input < pixel->input_num; input++) {
AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input));
}
}
void CollectComputeInputs(const ShaderComputeInputInfo* compute, ShaderInfo& info) {
if (compute != nullptr) {
if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) {
AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID");
}
if (compute->thread_ids_num > 0) {
AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID");
}
if (compute->thread_ids_num > 0 || compute->tg_size_en) {
AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex");
}
if (compute->dispatch_thread_dimensions) {
AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID");
}
}
}
void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::GetBuiltin) {
continue;
}
const auto kind = static_cast<StageInputKind>(inst.Arg(0).U32());
switch (kind) {
case StageInputKind::VertexIndex:
AddInput(info, kind, 0, 1, "gl_VertexIndex");
break;
case StageInputKind::InstanceIndex:
AddInput(info, kind, 0, 1, "gl_InstanceIndex");
break;
case StageInputKind::FragCoord: AddInput(info, kind, 0, 4, "gl_FragCoord"); break;
case StageInputKind::FrontFacing:
AddInput(info, kind, 0, 1, "gl_FrontFacing");
break;
case StageInputKind::WorkgroupId:
AddInput(info, kind, 0, 3, "gl_WorkGroupID");
break;
case StageInputKind::LocalInvocationId:
AddInput(info, kind, 0, 3, "gl_LocalInvocationID");
break;
case StageInputKind::LocalInvocationIndex:
AddInput(info, kind, 0, 1, "gl_LocalInvocationIndex");
break;
case StageInputKind::GlobalInvocationId:
AddInput(info, kind, 0, 3, "gl_GlobalInvocationID");
break;
case StageInputKind::Parameter: break;
}
}
}
}
void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::SetAttribute) {
continue;
}
const auto& export_info = program.values->export_info[inst.Flags<ExportFlags>().index];
if (export_info.kind == ExportTargetKind::MrtZ) {
if (pixel != nullptr && (export_info.en & 0x1u) != 0 &&
pixel->ps_depth_export_enable) {
AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth");
}
if (pixel != nullptr && (export_info.en & 0x4u) != 0 &&
pixel->ps_sample_mask_export_enable) {
AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask");
}
continue;
}
if (export_info.en == 0) {
continue;
}
switch (export_info.kind) {
case ExportTargetKind::Position:
AddOutput(info, StageOutputKind::Position, export_info.index, 0,
"out_position");
break;
case ExportTargetKind::Parameter:
AddOutput(info, StageOutputKind::Parameter, export_info.index,
export_info.index, fmt::format("out_param_{}", export_info.index));
break;
case ExportTargetKind::Mrt:
AddOutput(info, StageOutputKind::Mrt, export_info.index, export_info.index,
fmt::format("out_mrt_{}", export_info.index));
break;
default: break;
}
}
}
}
} // namespace
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) {
if (!program.resource_tracking_complete || program.shader_info_complete) {
if (error != nullptr) {
*error = !program.resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected";
}
return false;
}
if (program.values == nullptr) {
if (error != nullptr) {
*error = "typed value program is not available";
}
return false;
}
if (!ValidateOptions(program, options, error)) {
return false;
}
if (!ValidateValueReferences(program, options, error)) {
return false;
}
auto next = program.info;
next.inputs.clear();
next.outputs.clear();
next.has_bitwise_xor = std::any_of(
program.values->blocks.begin(), program.values->blocks.end(), [](const auto* block) {
return std::any_of(block->begin(), block->end(), [](const auto& inst) {
return inst.GetOpcode() == ValueOpcode::BitwiseXor32;
});
});
switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break;
case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(options.compute, next); break;
default: return false;
}
CollectBuiltinInputs(program, next);
CollectOutputs(program, options.pixel, next);
program.info = std::move(next);
program.shader_info_complete = true;
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,20 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct ShaderInfoOptions {
const ShaderVertexInputInfo* vertex = nullptr;
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
};
// Completes the immutable shader interface after resource tracking. On failure Program::info and
// all completion state remain unchanged.
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_ */
@@ -0,0 +1,951 @@
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "graphics/shader/recompiler/ir/ValueProgram.h"
#include <algorithm>
#include <bit>
#include <cstring>
#include <fmt/format.h>
#include <unordered_map>
#include <unordered_set>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) {
return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
StageName(program.stage), pc, message);
}
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
if (base > AddressMask) {
return false;
}
if (offset < 0) {
const auto magnitude = uint64_t {0} - static_cast<uint64_t>(offset);
if (magnitude > base) {
return false;
}
result = base - magnitude;
return true;
}
const auto magnitude = static_cast<uint64_t>(offset);
if (magnitude > AddressMask - base) {
return false;
}
result = base + magnitude;
return true;
}
bool IsRawRead(const ValueProgram& values, const Inst& inst) {
if (inst.GetOpcode() != ValueOpcode::LoadAddressU32 &&
inst.GetOpcode() != ValueOpcode::ReadConstBuffer) {
return false;
}
const auto index = inst.Flags<MemoryFlags>().index;
return index < values.memory_info.size() &&
values.memory_info[index].kind == ResourceKind::ScalarBuffer;
}
bool IsDescriptorHandle(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::GetBufferResource:
case ValueOpcode::GetAddressResource:
case ValueOpcode::GetImageResource:
case ValueOpcode::GetSamplerResource: return true;
default: return false;
}
}
bool IsRuntimeIntegerOp(ValueOpcode op) {
switch (op) {
case ValueOpcode::CompositeConstructU32x2:
case ValueOpcode::CompositeExtractU32x2:
case ValueOpcode::PackUint2x32:
case ValueOpcode::UnpackUint2x32:
case ValueOpcode::BitFieldInsert:
case ValueOpcode::BitFieldUExtract:
case ValueOpcode::BitFieldSExtract:
case ValueOpcode::IAdd32:
case ValueOpcode::IAdd64:
case ValueOpcode::IAddCarry32:
case ValueOpcode::ISub32:
case ValueOpcode::ISub64:
case ValueOpcode::IMul32:
case ValueOpcode::IMul64:
case ValueOpcode::ShiftLeftLogical32:
case ValueOpcode::ShiftLeftLogical64:
case ValueOpcode::ShiftRightLogical32:
case ValueOpcode::ShiftRightLogical64:
case ValueOpcode::ShiftRightArithmetic32:
case ValueOpcode::ShiftRightArithmetic64:
case ValueOpcode::BitwiseAnd32:
case ValueOpcode::BitwiseAnd64:
case ValueOpcode::BitwiseOr32:
case ValueOpcode::BitwiseOr64:
case ValueOpcode::BitwiseXor32:
case ValueOpcode::BitwiseNot32:
case ValueOpcode::SelectU1:
case ValueOpcode::SelectU32:
case ValueOpcode::ULessThan32:
case ValueOpcode::IEqual32:
case ValueOpcode::UGreaterThan32:
case ValueOpcode::INotEqual32:
case ValueOpcode::LogicalOr:
case ValueOpcode::LogicalAnd:
case ValueOpcode::LogicalXor:
case ValueOpcode::LogicalNot: return true;
default: return false;
}
}
class RuntimeValidator {
public:
RuntimeValidator(const Program& program, const ValueProgram& values)
: m_program(program), m_values(values) {}
bool Run(Value value, std::string& reason) { return Validate(value, reason); }
private:
bool Validate(Value value, std::string& reason) {
value = value.Resolve();
const auto* inst = value.TryInstruction();
if (inst == nullptr) {
switch (value.GetType()) {
case Type::U1:
case Type::U8:
case Type::U16:
case Type::U32:
case Type::U64: return true;
default:
reason = fmt::format("contains a non-integer immediate of type {}",
TypeName(value.GetType()));
return false;
}
}
if (!m_visiting.insert(inst).second) {
reason = fmt::format("contains a cyclic {} value", ValueOpcodeName(inst->GetOpcode()));
return false;
}
const auto finish = [&](bool valid) {
m_visiting.erase(inst);
return valid;
};
const auto op = inst->GetOpcode();
if (op == ValueOpcode::UndefU1 || op == ValueOpcode::UndefU8 ||
op == ValueOpcode::UndefU16 || op == ValueOpcode::UndefU32 ||
op == ValueOpcode::UndefU64 || op == ValueOpcode::Void) {
reason = fmt::format("contains {}", ValueOpcodeName(op));
return finish(false);
}
if (op == ValueOpcode::GetUserData) {
if (inst->NumArgs() != 1 || inst->Arg(0).GetType() != Type::ScalarReg) {
reason = "contains a malformed GetUserData";
return finish(false);
}
const auto reg = RegIndex(inst->Arg(0).ScalarRegister());
if (reg < m_program.user_data_base ||
reg - m_program.user_data_base >= m_program.user_data_count) {
reason = fmt::format("references unavailable user SGPR {}", reg);
return finish(false);
}
return finish(true);
}
if (op == ValueOpcode::GetShaderBase) {
if (inst->NumArgs() != 0) {
reason = "contains a malformed GetShaderBase";
return finish(false);
}
return finish(true);
}
if (op == ValueOpcode::Phi) {
const auto invariant = ResolveInvariantPhi(m_values, value);
if (invariant.IsEmpty()) {
reason = "contains a control-dependent phi";
return finish(false);
}
return finish(Validate(invariant, reason));
}
if (op == ValueOpcode::GetSrtResource) {
if (inst->NumArgs() != 0) {
reason = "contains a malformed GetSrtResource";
return finish(false);
}
return finish(true);
}
if (op == ValueOpcode::ReadConst) {
const auto slot = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
if (inst->NumArgs() != 2 || inst->Arg(0).Resolve().TryInstruction() == nullptr ||
inst->Arg(0).Resolve().TryInstruction()->GetOpcode() !=
ValueOpcode::GetSrtResource ||
!slot.IsImmediate() || slot.GetType() != Type::U32 ||
slot.U32() >= m_values.srt_reads.size()) {
reason = "contains a malformed flattened SRT read";
return finish(false);
}
} else if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
const auto expected = op == ValueOpcode::LoadAddressU32
? ValueOpcode::GetAddressResource
: ValueOpcode::GetBufferResource;
const auto* handle = inst->NumArgs() != 0 ? inst->Arg(0).ResolveInstruction() : nullptr;
if (!IsRawRead(m_values, *inst) || handle == nullptr ||
handle->GetOpcode() != expected) {
reason = fmt::format("contains a non-scalar {}", ValueOpcodeName(op));
return finish(false);
}
} else if (op == ValueOpcode::PackUint2x32) {
const auto* pair = inst->NumArgs() == 1 ? inst->Arg(0).ResolveInstruction() : nullptr;
if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2) {
reason = "contains an unsupported packed runtime value";
return finish(false);
}
} else if (op == ValueOpcode::CompositeExtractU32x2) {
const auto* source = inst->NumArgs() == 2 ? inst->Arg(0).ResolveInstruction() : nullptr;
const auto index = inst->NumArgs() == 2 ? inst->Arg(1).Resolve() : Value {};
if (source == nullptr || !index.IsImmediate() || index.GetType() != Type::U32 ||
index.U32() >= 2u ||
(source->GetOpcode() != ValueOpcode::CompositeConstructU32x2 &&
source->GetOpcode() != ValueOpcode::UnpackUint2x32 &&
source->GetOpcode() != ValueOpcode::IAddCarry32)) {
reason = "contains an unsupported composite runtime source";
return finish(false);
}
}
if (op == ValueOpcode::GetBufferResource || op == ValueOpcode::GetImageResource ||
op == ValueOpcode::GetSamplerResource || op == ValueOpcode::GetAddressResource) {
const size_t expected = op == ValueOpcode::GetBufferResource ? 4u
: op == ValueOpcode::GetImageResource ? 8u
: op == ValueOpcode::GetSamplerResource ? 4u
: 2u;
if (inst->NumArgs() != expected) {
reason = fmt::format("contains a malformed {}", ValueOpcodeName(op));
return finish(false);
}
} else if (op != ValueOpcode::ReadConst && op != ValueOpcode::ReadConstBuffer &&
op != ValueOpcode::LoadAddressU32 && !IsRuntimeIntegerOp(op)) {
reason =
fmt::format("contains unsupported or control-dependent {}", ValueOpcodeName(op));
return finish(false);
}
for (size_t index = 0; index < inst->NumArgs(); index++) {
if (!Validate(inst->Arg(index), reason)) {
return finish(false);
}
}
return finish(true);
}
const Program& m_program;
const ValueProgram& m_values;
std::unordered_set<const Inst*> m_visiting;
};
class PlanBuilder {
public:
PlanBuilder(Program& program, ValueProgram& values): m_program(program), m_values(values) {}
bool Run(std::string* error) {
m_values.srt_reads.clear();
m_values.dynamic_reads.clear();
for (auto* block: m_values.blocks) {
for (auto& inst: *block) {
if (IsDescriptorHandle(inst.GetOpcode())) {
for (size_t index = 0; index < inst.NumArgs(); index++) {
if (!Collect(inst.Arg(index), 0, error)) {
return false;
}
}
}
}
}
for (auto* block: m_values.blocks) {
for (auto& inst: *block) {
std::string reason;
if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_values, inst) &&
inst.Arg(1).Resolve().IsImmediate() &&
ValidateRuntimeValue(m_program, Value(&inst), reason) &&
!Collect(Value(&inst), inst.Flags<MemoryFlags>().pc, error)) {
return false;
}
}
}
PatchReads();
return true;
}
private:
struct Patch {
Inst* inst = nullptr;
uint32_t slot = 0;
bool keep = false;
};
bool Fail(uint32_t pc, std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, pc, message);
}
return false;
}
bool Collect(Value value, uint32_t use_pc, std::string* error) {
value = value.Resolve();
if (value.IsImmediate()) {
return true;
}
auto* inst = value.TryInstruction();
if (inst == nullptr) {
return Fail(use_pc, error, "invalid typed planning value");
}
if (std::ranges::find(m_visiting, inst) != m_visiting.end()) {
return inst->GetOpcode() == ValueOpcode::Phi;
}
if (std::ranges::find(m_visited, inst) != m_visited.end()) {
return true;
}
m_visiting.push_back(inst);
for (size_t index = 0; index < inst->NumArgs(); index++) {
if (!Collect(inst->Arg(index), use_pc, error)) {
return false;
}
}
m_visiting.pop_back();
m_visited.push_back(inst);
if (!IsRawRead(m_values, *inst)) {
return true;
}
const auto offset = inst->Arg(1).Resolve();
if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
if (std::ranges::find(m_values.dynamic_reads, value) == m_values.dynamic_reads.end()) {
m_values.dynamic_reads.push_back(value);
}
return true;
}
use_pc = inst->Flags<MemoryFlags>().pc;
for (uint32_t slot = 0; slot < m_values.srt_reads.size(); slot++) {
if (EquivalentValue(m_values, value, m_values.srt_reads[slot].value)) {
m_patches.push_back({inst, slot, false});
return true;
}
}
const auto slot = static_cast<uint32_t>(m_values.srt_reads.size());
m_values.srt_reads.push_back({value, slot, use_pc});
m_patches.push_back({inst, slot, true});
return true;
}
void PatchReads() {
for (const auto& patch: m_patches) {
auto* block = patch.inst->Parent();
auto& list = block->Instructions();
auto where =
std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; });
const auto resource =
Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource));
const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst,
{resource, Value(patch.slot)}));
const auto uses = patch.inst->Uses();
for (const auto& use: uses) {
use.user->SetArg(use.operand, flat);
}
for (auto& info: m_values.block_info) {
if (info.condition.Resolve() == Value(patch.inst)) {
info.condition = flat;
}
if (info.indirect_target.Resolve() == Value(patch.inst)) {
info.indirect_target = flat;
}
}
if (patch.keep) {
const auto memory = patch.inst->Flags<MemoryFlags>().index;
if (memory < m_values.memory_info.size()) {
m_values.memory_info[memory].planning_only = true;
}
block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
}
}
}
Program& m_program;
ValueProgram& m_values;
std::vector<Inst*> m_visiting;
std::vector<Inst*> m_visited;
std::vector<Patch> m_patches;
};
class Evaluator {
public:
Evaluator(const Program& program, const SrtRuntime& runtime)
: m_program(program), m_values(*program.values), m_runtime(runtime) {}
void SetUsePc(uint32_t pc) { m_use_pc = pc; }
bool Evaluate(Value value, uint32_t& result, std::string* error) {
uint64_t wide = 0;
if (!EvaluateWide(value, wide, error)) {
return false;
}
result = static_cast<uint32_t>(wide);
return true;
}
private:
bool Fail(std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, m_use_pc, message);
}
return false;
}
bool EvaluateWide(Value value, uint64_t& result, std::string* error) {
value = value.Resolve();
if (value.IsImmediate()) {
switch (value.GetType()) {
case Type::U1: result = value.U1(); return true;
case Type::U8: result = value.U8(); return true;
case Type::U16: result = value.U16(); return true;
case Type::U32: result = value.U32(); return true;
case Type::U64: result = value.U64(); return true;
default: return Fail(error, "non-integer immediate in runtime expression");
}
}
auto* inst = value.TryInstruction();
if (inst == nullptr) {
return Fail(error, "invalid typed runtime value");
}
if (const auto found = m_cache.find(inst); found != m_cache.end()) {
result = found->second;
return true;
}
if (std::ranges::find(m_visiting, inst) != m_visiting.end()) {
return Fail(error, "cyclic typed runtime value");
}
m_visiting.push_back(inst);
uint64_t out = 0;
if (!EvaluateInst(*inst, out, error)) {
return false;
}
m_visiting.pop_back();
m_cache.emplace(inst, out);
result = out;
return true;
}
bool Arg(const Inst& inst, size_t index, uint64_t& result, std::string* error) {
return EvaluateWide(inst.Arg(index), result, error);
}
bool EvaluatePhi(const Inst& inst, uint64_t& result, std::string* error) {
const auto value = ResolveInvariantPhi(m_values, Value(const_cast<Inst*>(&inst)));
return !value.IsEmpty() ? EvaluateWide(value, result, error)
: Fail(error, "typed phi has runtime-dependent values");
}
bool EvaluateExtract(const Inst& inst, uint64_t& result, std::string* error) {
const auto index = inst.Arg(1).Resolve();
if (!index.IsImmediate() || index.GetType() != Type::U32) {
return Fail(error, "dynamic composite extract in runtime expression");
}
const auto component = index.U32();
const auto* source = inst.Arg(0).ResolveInstruction();
if (source == nullptr || component >= 2u) {
return Fail(error, "unsupported composite runtime source");
}
if (source->GetOpcode() == ValueOpcode::CompositeConstructU32x2) {
return EvaluateWide(source->Arg(component), result, error);
}
if (source->GetOpcode() == ValueOpcode::UnpackUint2x32) {
uint64_t packed = 0;
if (!Arg(*source, 0, packed, error)) {
return false;
}
result = static_cast<uint32_t>(packed >> (component * 32u));
return true;
}
if (source->GetOpcode() == ValueOpcode::IAddCarry32) {
uint64_t lhs = 0;
uint64_t rhs = 0;
if (!Arg(*source, 0, lhs, error) || !Arg(*source, 1, rhs, error)) {
return false;
}
const auto sum =
static_cast<uint64_t>(static_cast<uint32_t>(lhs)) + static_cast<uint32_t>(rhs);
result =
component == 0u ? static_cast<uint32_t>(sum) : static_cast<uint32_t>(sum >> 32u);
return true;
}
return Fail(error, "unsupported composite runtime source");
}
bool EvaluateRawRead(const Inst& inst, uint64_t& result, std::string* error) {
const auto flags = inst.Flags<MemoryFlags>();
if (flags.index >= m_values.memory_info.size()) {
return Fail(error, "raw scalar read has invalid metadata");
}
const auto& mem = m_values.memory_info[flags.index];
const auto* handle = inst.Arg(0).ResolveInstruction();
if (handle == nullptr) {
return Fail(error, "raw scalar read has no descriptor handle");
}
uint64_t low = 0;
uint64_t high = 0;
uint64_t offset = 0;
if (!Arg(*handle, 0, low, error) || !Arg(*handle, 1, high, error) ||
!Arg(inst, 1, offset, error)) {
return false;
}
const auto base = ((high << 32u) | static_cast<uint32_t>(low)) & AddressMask;
const auto immediate = static_cast<int64_t>(static_cast<int32_t>(mem.offset));
uint64_t address = 0;
if (inst.GetOpcode() == ValueOpcode::ReadConstBuffer) {
uint64_t records = 0;
uint64_t word3 = 0;
if (handle->NumArgs() != 4u || !Arg(*handle, 2, records, error) ||
!Arg(*handle, 3, word3, error)) {
return Fail(error, "constant-buffer descriptor has invalid width");
}
if (immediate < 0) {
return Fail(error, "constant-buffer read has a negative immediate offset");
}
const auto byte_offset =
static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
const auto aligned = byte_offset & ~uint64_t {3};
const auto stride = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
const auto size = stride == 0u
? static_cast<uint64_t>(static_cast<uint32_t>(records))
: static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
if (aligned > size || size - aligned < sizeof(uint32_t)) {
return Fail(
error, fmt::format("constant-buffer offset {} exceeds size {}", aligned, size));
}
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
} else {
const auto relative = (immediate & ~int64_t {3}) +
static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
if (!AddSignedAddress(base & ~uint64_t {3}, relative, address)) {
return Fail(error, "raw scalar read is outside the 48-bit address space");
}
}
uint32_t word = 0;
if (m_runtime.read_memory != nullptr) {
if (!m_runtime.read_memory(m_runtime.userdata, address, &word)) {
return Fail(error, fmt::format("constant read failed at 0x{:016x}", address));
}
} else {
std::memcpy(&word, reinterpret_cast<const void*>(address), sizeof(word));
}
result = word;
return true;
}
bool EvaluateInst(const Inst& inst, uint64_t& result, std::string* error) {
uint64_t a = 0;
uint64_t b = 0;
uint64_t c = 0;
const auto binary = [&]() { return Arg(inst, 0, a, error) && Arg(inst, 1, b, error); };
const auto ternary = [&]() {
return Arg(inst, 0, a, error) && Arg(inst, 1, b, error) && Arg(inst, 2, c, error);
};
switch (inst.GetOpcode()) {
case ValueOpcode::GetUserData: {
const auto reg = RegIndex(inst.Arg(0).ScalarRegister());
if (reg < m_program.user_data_base ||
reg - m_program.user_data_base >= m_runtime.user_data.size()) {
return Fail(error, fmt::format("user SGPR {} is unavailable", reg));
}
result = m_runtime.user_data[reg - m_program.user_data_base];
return true;
}
case ValueOpcode::GetShaderBase: result = m_runtime.shader_base; return true;
case ValueOpcode::Phi: return EvaluatePhi(inst, result, error);
case ValueOpcode::CompositeExtractU32x2: return EvaluateExtract(inst, result, error);
case ValueOpcode::PackUint2x32: {
const auto* pair = inst.Arg(0).ResolveInstruction();
if (pair == nullptr || pair->GetOpcode() != ValueOpcode::CompositeConstructU32x2 ||
!Arg(*pair, 0, a, error) || !Arg(*pair, 1, b, error)) {
return Fail(error, "unsupported packed runtime value");
}
result = static_cast<uint32_t>(a) |
(static_cast<uint64_t>(static_cast<uint32_t>(b)) << 32u);
return true;
}
case ValueOpcode::ReadConst: {
const auto slot = inst.Arg(1).Resolve();
if (!slot.IsImmediate() || slot.GetType() != Type::U32 ||
slot.U32() >= m_values.srt_reads.size()) {
return Fail(error, "invalid flattened SRT index");
}
return EvaluateWide(m_values.srt_reads[slot.U32()].value, result, error);
}
case ValueOpcode::LoadAddressU32:
case ValueOpcode::ReadConstBuffer:
if (IsRawRead(m_values, inst)) {
return EvaluateRawRead(inst, result, error);
}
break;
case ValueOpcode::IAdd32:
if (binary()) {
result = static_cast<uint32_t>(a + b);
return true;
}
return false;
case ValueOpcode::IAdd64:
if (binary()) {
result = a + b;
return true;
}
return false;
case ValueOpcode::ISub32:
if (binary()) {
result = static_cast<uint32_t>(a - b);
return true;
}
return false;
case ValueOpcode::ISub64:
if (binary()) {
result = a - b;
return true;
}
return false;
case ValueOpcode::IMul32:
if (binary()) {
result = static_cast<uint32_t>(a * b);
return true;
}
return false;
case ValueOpcode::IMul64:
if (binary()) {
result = a * b;
return true;
}
return false;
case ValueOpcode::BitwiseAnd32:
if (binary()) {
result = static_cast<uint32_t>(a & b);
return true;
}
return false;
case ValueOpcode::BitwiseAnd64:
if (binary()) {
result = a & b;
return true;
}
return false;
case ValueOpcode::BitwiseOr32:
if (binary()) {
result = static_cast<uint32_t>(a | b);
return true;
}
return false;
case ValueOpcode::BitwiseOr64:
if (binary()) {
result = a | b;
return true;
}
return false;
case ValueOpcode::BitwiseXor32:
if (binary()) {
result = static_cast<uint32_t>(a ^ b);
return true;
}
return false;
case ValueOpcode::BitwiseNot32:
if (Arg(inst, 0, a, error)) {
result = ~static_cast<uint32_t>(a);
return true;
}
return false;
case ValueOpcode::ShiftLeftLogical32:
if (binary()) {
result = static_cast<uint32_t>(a) << (b & 31u);
return true;
}
return false;
case ValueOpcode::ShiftLeftLogical64:
if (binary()) {
result = a << (b & 63u);
return true;
}
return false;
case ValueOpcode::ShiftRightLogical32:
if (binary()) {
result = static_cast<uint32_t>(a) >> (b & 31u);
return true;
}
return false;
case ValueOpcode::ShiftRightLogical64:
if (binary()) {
result = a >> (b & 63u);
return true;
}
return false;
case ValueOpcode::ShiftRightArithmetic32:
if (binary()) {
result = static_cast<uint32_t>(
std::bit_cast<int32_t>(static_cast<uint32_t>(a)) >> (b & 31u));
return true;
}
return false;
case ValueOpcode::ShiftRightArithmetic64:
if (binary()) {
result = static_cast<uint64_t>(std::bit_cast<int64_t>(a) >> (b & 63u));
return true;
}
return false;
case ValueOpcode::BitFieldUExtract:
if (ternary()) {
const auto offset = static_cast<uint32_t>(b);
const auto width = static_cast<uint32_t>(c);
if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid unsigned bit-field range");
}
const auto mask = width == 32u ? UINT32_MAX
: width == 0u ? 0u
: (uint32_t {1} << width) - 1u;
result = width == 0u ? 0u : (static_cast<uint32_t>(a) >> offset) & mask;
return true;
}
return false;
case ValueOpcode::BitFieldSExtract:
if (ternary()) {
const auto offset = static_cast<uint32_t>(b);
const auto width = static_cast<uint32_t>(c);
if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid signed bit-field range");
}
if (width == 0u) {
result = 0;
return true;
}
const auto mask = width == 32u ? UINT32_MAX : (uint32_t {1} << width) - 1u;
auto bits = (static_cast<uint32_t>(a) >> offset) & mask;
if (width < 32u && (bits & (uint32_t {1} << (width - 1u))) != 0u) {
bits |= ~mask;
}
result = bits;
return true;
}
return false;
case ValueOpcode::BitFieldInsert: {
uint64_t d = 0;
if (!ternary() || !Arg(inst, 3, d, error)) {
return false;
}
const auto offset = static_cast<uint32_t>(c);
const auto width = static_cast<uint32_t>(d);
if (offset > 32u || width > 32u - offset) {
return Fail(error, "invalid inserted bit-field range");
}
if (width == 0u) {
result = static_cast<uint32_t>(a);
return true;
}
const auto mask =
width == 32u ? UINT32_MAX : ((uint32_t {1} << width) - 1u) << offset;
result = (static_cast<uint32_t>(a) & ~mask) |
((static_cast<uint32_t>(b) << offset) & mask);
return true;
}
case ValueOpcode::SelectU32:
case ValueOpcode::SelectU1:
if (ternary()) {
result = a != 0u ? b : c;
return true;
}
return false;
case ValueOpcode::IEqual32:
if (binary()) {
result = static_cast<uint32_t>(a) == static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::INotEqual32:
if (binary()) {
result = static_cast<uint32_t>(a) != static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::ULessThan32:
if (binary()) {
result = static_cast<uint32_t>(a) < static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::UGreaterThan32:
if (binary()) {
result = static_cast<uint32_t>(a) > static_cast<uint32_t>(b);
return true;
}
return false;
case ValueOpcode::LogicalAnd:
if (binary()) {
result = (a != 0u) && (b != 0u);
return true;
}
return false;
case ValueOpcode::LogicalOr:
if (binary()) {
result = (a != 0u) || (b != 0u);
return true;
}
return false;
case ValueOpcode::LogicalXor:
if (binary()) {
result = (a != 0u) != (b != 0u);
return true;
}
return false;
case ValueOpcode::LogicalNot:
if (Arg(inst, 0, a, error)) {
result = a == 0u;
return true;
}
return false;
case ValueOpcode::UndefU1:
case ValueOpcode::UndefU8:
case ValueOpcode::UndefU16:
case ValueOpcode::UndefU32:
case ValueOpcode::UndefU64: return Fail(error, "undefined typed runtime value");
default: break;
}
return Fail(error, fmt::format("unsupported typed runtime opcode {}",
ValueOpcodeName(inst.GetOpcode())));
}
const Program& m_program;
const ValueProgram& m_values;
const SrtRuntime& m_runtime;
uint32_t m_use_pc = 0;
std::unordered_map<const Inst*, uint64_t> m_cache;
std::vector<const Inst*> m_visiting;
};
const DescriptorSource* Source(const Program& program, uint32_t source) {
if (program.values == nullptr || source >= program.values->descriptor_sources.size()) {
return nullptr;
}
return &program.values->descriptor_sources[source];
}
bool EvaluateRuntimeSourcesImpl(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, bool evaluate_flat,
std::string* error) {
if (program.values == nullptr || !program.srt_plan_complete) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "typed SRT plan is not ready");
}
return false;
}
Evaluator evaluator(program, runtime);
std::vector<DescriptorValue> evaluated;
evaluated.reserve(requests.size());
for (const auto& request: requests) {
const auto* source = Source(program, request.source);
if (source == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, request.use_pc,
fmt::format("invalid descriptor source {}", request.source));
}
return false;
}
evaluator.SetUsePc(request.use_pc);
DescriptorValue value;
value.dword_count = source->dword_count;
for (uint32_t index = 0; index < source->dword_count; index++) {
if (!evaluator.Evaluate(source->dwords[index], value.dwords[index], error)) {
return false;
}
}
evaluated.push_back(value);
}
std::vector<uint32_t> flattened;
if (evaluate_flat) {
flattened.resize(program.values->srt_reads.size());
for (const auto& read: program.values->srt_reads) {
evaluator.SetUsePc(read.use_pc);
if (read.flat_offset >= flattened.size() ||
!evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) {
return false;
}
}
}
results = std::move(evaluated);
if (evaluate_flat) {
flat = std::move(flattened);
}
return true;
}
} // namespace
bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason) {
if (program.values == nullptr) {
reason = "typed SSA is not ready";
return false;
}
return RuntimeValidator(program, *program.values).Run(value, reason);
}
bool BuildSrtPlan(Program& program, std::string* error) {
if (program.values == nullptr || program.resource_tracking_complete) {
if (error != nullptr) {
*error = program.values == nullptr ? "typed SSA is not ready"
: "cannot rebuild SRT after resource tracking";
}
return false;
}
program.srt_plan_complete = false;
if (!PlanBuilder(program, *program.values).Run(error)) {
return false;
}
program.srt_plan_complete = true;
return true;
}
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue& result,
std::string* error) {
const DescriptorSourceRequest request {source, use_pc};
std::vector<DescriptorValue> results;
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) {
return false;
}
result = results.front();
return true;
}
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::string* error) {
std::vector<uint32_t> ignored;
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error);
}
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::string* error) {
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error);
}
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
std::string* error) {
std::vector<DescriptorValue> ignored;
return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,56 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include <optional>
#include <span>
namespace Libs::Graphics::ShaderRecompiler::IR {
class Value;
using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* value);
struct SrtRuntime {
std::span<const uint32_t> user_data;
uint64_t shader_base = 0;
SrtMemoryReader read_memory = nullptr;
void* userdata = nullptr;
std::optional<uint64_t> flat_memory_base;
};
struct DescriptorSourceRequest {
uint32_t source = 0;
uint32_t use_pc = 0;
};
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
// dynamic offsets remain explicit and are never assigned a fake slot.
bool BuildSrtPlan(Program& program, std::string* error);
bool ValidateRuntimeValue(const Program& program, Value value, std::string& reason);
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue& result,
std::string* error);
// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by
// several descriptors are memoized once across the batch.
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::string* error);
// Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk.
// On failure neither destination is changed.
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::string* error);
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ */
@@ -0,0 +1,377 @@
#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
#include <algorithm>
#include <map>
#include <unordered_map>
#include <variant>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
struct SccTag {
auto operator<=>(const SccTag&) const = default;
};
struct ThreadBitScalarReg {
ScalarReg reg {};
auto operator<=>(const ThreadBitScalarReg&) const = default;
};
struct ExecTag {
auto operator<=>(const ExecTag&) const = default;
};
struct ExecLoTag {
auto operator<=>(const ExecLoTag&) const = default;
};
struct ExecHiTag {
auto operator<=>(const ExecHiTag&) const = default;
};
struct VccTag {
auto operator<=>(const VccTag&) const = default;
};
struct VccLoTag {
auto operator<=>(const VccLoTag&) const = default;
};
struct VccHiTag {
auto operator<=>(const VccHiTag&) const = default;
};
struct M0Tag {
auto operator<=>(const M0Tag&) const = default;
};
struct GotoVariable {
uint32_t index = 0;
auto operator<=>(const GotoVariable&) const = default;
};
using Variable = std::variant<ScalarReg, ThreadBitScalarReg, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using ValueMap = std::unordered_map<Block*, Value>;
struct DefTable {
const Value& Get(Block* block, ScalarReg reg) {
if (RegIndex(reg) >= NumScalarRegs) {
EXIT("SSA scalar read index is out of range: %u", RegIndex(reg));
}
return block->ssa_sreg_values[RegIndex(reg)];
}
void Set(Block* block, ScalarReg reg, Value value) {
if (RegIndex(reg) >= NumScalarRegs) {
EXIT("SSA scalar write index is out of range: %u", RegIndex(reg));
}
block->ssa_sreg_values[RegIndex(reg)] = value;
}
const Value& Get(Block* block, ThreadBitScalarReg value) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
return block->ssa_thread_bit_sreg_values[RegIndex(value.reg)];
}
void Set(Block* block, ThreadBitScalarReg value, Value definition) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, VectorReg reg) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
return block->ssa_vreg_values[RegIndex(reg)];
}
void Set(Block* block, VectorReg reg, Value value) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
block->ssa_vreg_values[RegIndex(reg)] = value;
}
const Value& Get(Block* block, GotoVariable value) {
return goto_variables[value.index][block];
}
void Set(Block* block, GotoVariable value, Value definition) {
goto_variables[value.index][block] = definition;
}
const Value& Get(Block* block, SccTag) { return scc[block]; }
void Set(Block* block, SccTag, Value value) { scc[block] = value; }
const Value& Get(Block* block, ExecTag) { return exec[block]; }
void Set(Block* block, ExecTag, Value value) { exec[block] = value; }
const Value& Get(Block* block, ExecLoTag) { return exec_lo[block]; }
void Set(Block* block, ExecLoTag, Value value) { exec_lo[block] = value; }
const Value& Get(Block* block, ExecHiTag) { return exec_hi[block]; }
void Set(Block* block, ExecHiTag, Value value) { exec_hi[block] = value; }
const Value& Get(Block* block, VccTag) { return vcc[block]; }
void Set(Block* block, VccTag, Value value) { vcc[block] = value; }
const Value& Get(Block* block, VccLoTag) { return vcc_lo[block]; }
void Set(Block* block, VccLoTag, Value value) { vcc_lo[block] = value; }
const Value& Get(Block* block, VccHiTag) { return vcc_hi[block]; }
void Set(Block* block, VccHiTag, Value value) { vcc_hi[block] = value; }
const Value& Get(Block* block, M0Tag) { return m0[block]; }
void Set(Block* block, M0Tag, Value value) { m0[block] = value; }
ValueMap scc;
ValueMap exec;
ValueMap exec_lo;
ValueMap exec_hi;
ValueMap vcc;
ValueMap vcc_lo;
ValueMap vcc_hi;
ValueMap m0;
std::unordered_map<uint32_t, ValueMap> goto_variables;
};
ValueOpcode UndefOpcode(ScalarReg) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(ThreadBitScalarReg) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VectorReg) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(GotoVariable) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(SccTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ExecTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ExecLoTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(ExecHiTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(VccTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VccLoTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(VccHiTag) {
return ValueOpcode::UndefU32;
}
ValueOpcode UndefOpcode(M0Tag) {
return ValueOpcode::UndefU32;
}
Value InitialValue(ScalarReg) {
return Value(0u);
}
Value InitialValue(ThreadBitScalarReg) {
return Value(false);
}
Value InitialValue(VectorReg) {
return Value(0u);
}
Value InitialValue(GotoVariable) {
return Value(false);
}
Value InitialValue(SccTag) {
return Value(false);
}
Value InitialValue(ExecTag) {
return Value(false);
}
Value InitialValue(ExecLoTag) {
return Value(0u);
}
Value InitialValue(ExecHiTag) {
return Value(0u);
}
Value InitialValue(VccTag) {
return Value(false);
}
Value InitialValue(VccLoTag) {
return Value(0u);
}
Value InitialValue(VccHiTag) {
return Value(0u);
}
Value InitialValue(M0Tag) {
return Value(0u);
}
enum class ReadStep { Start, SetValue, PushPhiArgument };
template <typename T>
struct ReadState {
Block* block = nullptr;
Value result;
Inst* phi = nullptr;
size_t pred = 0;
ReadStep step = ReadStep::Start;
};
class Pass {
public:
template <typename T>
void Write(T variable, Block* block, Value value) {
definitions.Set(block, variable, value);
}
template <typename T>
Value Read(T variable, Block* root) {
std::vector<ReadState<T>> stack {{}, {.block = root}};
const auto prepare_phi = [&]() {
auto& state = stack.back();
const auto predecessors = state.block->ImmPredecessors();
if (state.pred == predecessors.size()) {
const auto result = TryRemoveTrivialPhi(*state.phi, InitialValue(variable));
const auto block = state.block;
stack.pop_back();
stack.back().result = result;
Write(variable, block, result);
return;
}
state.step = ReadStep::PushPhiArgument;
stack.push_back({.block = predecessors[state.pred]});
};
do {
auto& state = stack.back();
auto* block = state.block;
switch (state.step) {
case ReadStep::Start: {
if (const auto& def = definitions.Get(block, variable); !def.IsEmpty()) {
state.result = def;
} else if (!block->IsSsaSealed()) {
auto& phi = *block->PrependNewInst(block->begin(), ValueOpcode::Phi);
phi.SetFlags(TypeOf(UndefOpcode(variable)));
incomplete_phis[block][Variable(variable)] = &phi;
state.result = Value(&phi);
} else if (const auto predecessors = block->ImmPredecessors();
predecessors.size() == 1) {
state.step = ReadStep::SetValue;
stack.push_back({.block = predecessors.front()});
break;
} else {
auto& phi = *block->PrependNewInst(block->begin(), ValueOpcode::Phi);
phi.SetFlags(TypeOf(UndefOpcode(variable)));
Write(variable, block, Value(&phi));
state.phi = &phi;
prepare_phi();
break;
}
[[fallthrough]];
}
case ReadStep::SetValue: {
const auto result = state.result;
Write(variable, block, result);
stack.pop_back();
stack.back().result = result;
break;
}
case ReadStep::PushPhiArgument: {
const auto predecessors = block->ImmPredecessors();
state.phi->AddPhiOperand(predecessors[state.pred], state.result);
state.pred++;
prepare_phi();
break;
}
}
} while (stack.size() > 1);
return stack.back().result;
}
void Seal(Block* block) {
if (const auto found = incomplete_phis.find(block); found != incomplete_phis.end()) {
for (auto& [variable, phi]: found->second) {
std::visit([&](auto value) { AddPhiOperands(value, *phi, block); }, variable);
}
}
block->SsaSeal();
}
private:
template <typename T>
Value AddPhiOperands(T variable, Inst& phi, Block* block) {
for (auto* predecessor: block->ImmPredecessors()) {
phi.AddPhiOperand(predecessor, Read(variable, predecessor));
}
return TryRemoveTrivialPhi(phi, InitialValue(variable));
}
Value TryRemoveTrivialPhi(Inst& phi, Value initial_value) {
Value same;
for (size_t index = 0; index < phi.NumArgs(); index++) {
const auto operand = phi.Arg(index).Resolve();
if (operand == same.Resolve() || operand == Value(&phi)) {
continue;
}
if (!same.IsEmpty()) {
return Value(&phi);
}
same = operand;
}
if (same.IsEmpty()) {
same = initial_value;
}
const auto users = phi.Uses();
phi.ReplaceUsesWith(same);
for (const auto& use: users) {
if (use.user->GetOpcode() == ValueOpcode::Phi) {
TryRemoveTrivialPhi(*use.user, initial_value);
}
}
return same;
}
std::unordered_map<Block*, std::map<Variable, Inst*>> incomplete_phis;
DefTable definitions;
};
void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
switch (inst.GetOpcode()) {
case ValueOpcode::SetScalarRegister:
pass.Write(inst.Arg(0).ScalarRegister(), block, inst.Arg(1));
break;
case ValueOpcode::SetThreadBitScalarRegister:
pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetVectorRegister:
pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1));
break;
case ValueOpcode::SetGotoVariable:
pass.Write(GotoVariable {inst.Arg(0).U32()}, block, inst.Arg(1));
break;
case ValueOpcode::SetScc: pass.Write(SccTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetExec: pass.Write(ExecTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetExecLo: pass.Write(ExecLoTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetExecHi: pass.Write(ExecHiTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetVcc: pass.Write(VccTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetVccLo: pass.Write(VccLoTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetVccHi: pass.Write(VccHiTag {}, block, inst.Arg(0)); break;
case ValueOpcode::SetM0: pass.Write(M0Tag {}, block, inst.Arg(0)); break;
case ValueOpcode::GetScalarRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).ScalarRegister(), block));
break;
case ValueOpcode::GetThreadBitScalarRegister:
inst.ReplaceUsesWith(
pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetVectorRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block));
break;
case ValueOpcode::GetGotoVariable:
inst.ReplaceUsesWith(pass.Read(GotoVariable {inst.Arg(0).U32()}, block));
break;
case ValueOpcode::GetScc: inst.ReplaceUsesWith(pass.Read(SccTag {}, block)); break;
case ValueOpcode::GetExec: inst.ReplaceUsesWith(pass.Read(ExecTag {}, block)); break;
case ValueOpcode::GetExecLo: inst.ReplaceUsesWith(pass.Read(ExecLoTag {}, block)); break;
case ValueOpcode::GetExecHi: inst.ReplaceUsesWith(pass.Read(ExecHiTag {}, block)); break;
case ValueOpcode::GetVcc: inst.ReplaceUsesWith(pass.Read(VccTag {}, block)); break;
case ValueOpcode::GetVccLo: inst.ReplaceUsesWith(pass.Read(VccLoTag {}, block)); break;
case ValueOpcode::GetVccHi: inst.ReplaceUsesWith(pass.Read(VccHiTag {}, block)); break;
case ValueOpcode::GetM0: inst.ReplaceUsesWith(pass.Read(M0Tag {}, block)); break;
default: break;
}
}
} // namespace
void RewriteToSsa(const BlockList& blocks) {
Pass pass;
for (auto* block: blocks) {
for (auto& inst: *block) {
VisitInstruction(pass, block, inst);
}
}
for (auto* block: blocks) {
pass.Seal(block);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -0,0 +1,9 @@
#pragma once
#include "graphics/shader/recompiler/ir/Block.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void RewriteToSsa(const BlockList& blocks);
} // namespace Libs::Graphics::ShaderRecompiler::IR