Compare commits

...

2 Commits

Author SHA1 Message Date
nmzik 0791f921f6 UFC 5 in-game (#883)
* shader: release descriptor dependencies

* shader: simplify shared-exit control flow

* shader: fix scalar memory loads

* shader: fix LDS counter bounds

* renderer: skip inactive pixel shaders

* shader: respect FP32 mode in comparisons

* renderer: fix sampler border colors

* shader: resolve descriptors from control flow

* shader: resolve descriptors before resource planning

* shader: handle scalar guards for image bounds

* shader: handle reduced guards for image loops

* shader: fix wave guards and material indices

* shader: preserve scalar buffer offset alignment and carry

* shader: prove masked loop bounds across iteration transitions

* shader: preserve source and IR before resource tracking failures

* shader: track material keys selected from active lanes

* shader: remove obsolete post-dominator analysis

* shader: follow bounds guards through control-flow joins
2026-09-28 16:03:05 +04:00
nmzik 421684e75f docs: add weekly updates and Discord links to README (#866)
* docs: link weekly updates from README

* docs: add Discord development channel

* docs: use Discord server invite link

* docs: refine weekly updates wording
2026-09-27 18:06:26 +04:00
35 changed files with 3226 additions and 2261 deletions
+4
View File
@@ -7,6 +7,10 @@
[![Status](https://img.shields.io/badge/status-active%20development-orange.svg)](#current-status)
[![License](https://img.shields.io/badge/license-GPL--2.0-blue.svg)](LICENSE)
**[Weekly updates](https://github.com/KytyPS5/KytyPS5/discussions/862)** — game progress, recent fixes and ongoing development.
**[Development on Discord](https://discord.gg/UNrkMqGaBg)** — KytyPS5 development.
KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
with experimental macOS support. It is based on a heavily modified version of
[Kyty](https://github.com/InoriRus/Kyty). The project is in active development, and behavior
+9 -9
View File
@@ -14,10 +14,10 @@ SamplerCache::~SamplerCache() {
}
}
vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r, bool integer_border) {
Common::LockGuard lock(m_mutex);
const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]};
const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3], integer_border};
if (auto iter = m_samplers.find(key); iter != m_samplers.end()) {
return iter->second;
}
@@ -93,23 +93,23 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
return vk::SamplerAddressMode::eClampToBorder;
};
vk::BorderColor border = vk::BorderColor::eIntTransparentBlack;
vk::BorderColor border = integer_border ? vk::BorderColor::eIntTransparentBlack
: vk::BorderColor::eFloatTransparentBlack;
switch (static_cast<Prospero::SamplerBorderColor>(r.BorderColorType())) {
case Prospero::SamplerBorderColor::kTransBlack:
border = vk::BorderColor::eIntTransparentBlack;
break;
case Prospero::SamplerBorderColor::kTransBlack: break;
case Prospero::SamplerBorderColor::kOpaqueBlack:
border = vk::BorderColor::eIntOpaqueBlack;
border = integer_border ? vk::BorderColor::eIntOpaqueBlack
: vk::BorderColor::eFloatOpaqueBlack;
break;
case Prospero::SamplerBorderColor::kOpaqueWhite:
border = vk::BorderColor::eIntOpaqueWhite;
border = integer_border ? vk::BorderColor::eIntOpaqueWhite
: vk::BorderColor::eFloatOpaqueWhite;
break;
case Prospero::SamplerBorderColor::kFromTable:
LOGF(
"temporary: approximating table border color as transparent black, index = %" PRIu16
"\n",
r.BorderColorPtr());
border = vk::BorderColor::eIntTransparentBlack;
break;
default: EXIT("unknown border color: %d", static_cast<int>(r.BorderColorType()));
}
+2 -2
View File
@@ -24,10 +24,10 @@ public:
~SamplerCache();
KYTY_CLASS_NO_COPY(SamplerCache);
vk::Sampler GetSampler(const ShaderSamplerResource& r);
vk::Sampler GetSampler(const ShaderSamplerResource& r, bool integer_border);
private:
using SamplerKey = std::array<uint32_t, 4>;
using SamplerKey = std::array<uint32_t, 5>;
struct SamplerKeyHash {
std::size_t operator()(const SamplerKey& key) const {
@@ -738,14 +738,15 @@ static vk::Sampler NativeSampler(RenderContext& context,
const ShaderRecompiler::IR::CompiledShaderInfo& program,
uint32_t index,
const ShaderRecompiler::IR::DescriptorValue& value) {
auto descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
if (!program.info.samplers[index].depth_compare) {
auto descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
const auto& sampler = program.info.samplers[index];
if (!sampler.depth_compare) {
descriptor.fields[0] &= ~(0x7u << 12u);
}
if (program.info.samplers[index].force_point_filtering) {
if (sampler.force_point_filtering) {
descriptor.SetPointFiltering();
}
return context.GetSamplerCache().GetSampler(descriptor);
return context.GetSamplerCache().GetSampler(descriptor, sampler.integer_border);
}
static vk::DescriptorBufferInfo NativeUpload(RenderContext& context,
@@ -29,7 +29,6 @@
#include <span>
#include <spirv-tools/libspirv.hpp>
#include <string_view>
#include <tuple>
#include <utility>
#include <vector>
#include <xxhash.h>
@@ -116,33 +115,22 @@ void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
}
void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash,
std::span<const uint32_t> code, const std::string& decoded_dump) {
std::span<const uint32_t> code) {
if (!Config::GraphicsDebugDumpEnabled()) {
return;
}
EXIT_IF(code.empty());
static std::atomic_int id = 0;
const auto base = Config::GetShaderLogFolder() / "original" /
fmt::format("{:04d}_new_shader_{}_{:016x}", id++, stage_name, shader_hash);
Common::File::CreateDirectories(base.parent_path());
for (const auto& [suffix, data, size]: {
std::tuple {".bin", static_cast<const void*>(code.data()), code.size_bytes()},
std::tuple {".rdna2", static_cast<const void*>(decoded_dump.data()),
decoded_dump.size()},
}) {
if (size == 0) {
continue;
}
auto path = base;
path += suffix;
Common::File file(path);
if (file.IsInvalid()) {
const auto path_text = Common::PathToString(path);
LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
} else {
file.Write(data, size);
}
const auto path = Config::GetShaderLogFolder() / "original" /
fmt::format("{:04d}_new_shader_{}_{:016x}.bin", id++, stage_name, shader_hash);
Common::File::CreateDirectories(path.parent_path());
Common::File file(path);
if (file.IsInvalid()) {
const auto path_text = Common::PathToString(path);
LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
return;
}
file.Write(code.data(), code.size_bytes());
}
bool ValidateShaderSpirv(const char* label, uint64_t shader_hash,
@@ -225,25 +213,13 @@ struct PipelineCache::ProgramCache {
static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13;
Permutation CompilePermutation(const ShaderParams& params,
Permutation CompilePermutation(const char* stage_name,
const ShaderRecompiler::CompileOptions& options,
ShaderRecompiler::TranslateResult translated,
ShaderRecompiler::IR::ResourceSpecialization specialization,
uint32_t push_data_start_dword) {
const char* stage_name = nullptr;
switch (options.stage) {
case ShaderType::Vertex: stage_name = "vs"; break;
case ShaderType::Mesh: stage_name = "ms"; break;
case ShaderType::Local: stage_name = "ls"; break;
case ShaderType::TessellationControl: stage_name = "hs"; break;
case ShaderType::TessellationEvaluation: stage_name = "ds"; break;
case ShaderType::Pixel: stage_name = "ps"; break;
case ShaderType::Compute: stage_name = "cs"; break;
default: EXIT("invalid pipeline shader stage\n");
}
auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
specialization, push_data_start_dword);
DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump);
if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) {
DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label,
@@ -321,14 +297,15 @@ struct PipelineCache::ProgramCache {
stage_input.compute = &input_info;
}
const char* label = nullptr;
const char* stage_name = nullptr;
switch (stage) {
case ShaderType::Vertex: label = "ShaderRecompiler VS"; break;
case ShaderType::Mesh: label = "ShaderRecompiler MS"; break;
case ShaderType::Local: label = "ShaderRecompiler LS"; break;
case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break;
case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break;
case ShaderType::Pixel: label = "ShaderRecompiler PS"; break;
case ShaderType::Compute: label = "ShaderRecompiler CS"; break;
case ShaderType::Vertex: label = "ShaderRecompiler VS"; stage_name = "vs"; break;
case ShaderType::Mesh: label = "ShaderRecompiler MS"; stage_name = "ms"; break;
case ShaderType::Local: label = "ShaderRecompiler LS"; stage_name = "ls"; break;
case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; stage_name = "hs"; break;
case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; stage_name = "ds"; break;
case ShaderType::Pixel: label = "ShaderRecompiler PS"; stage_name = "ps"; break;
case ShaderType::Compute: label = "ShaderRecompiler CS"; stage_name = "cs"; break;
default: EXIT("invalid pipeline shader stage\n");
}
ShaderRecompiler::CompileOptions options;
@@ -351,6 +328,7 @@ struct PipelineCache::ProgramCache {
} else {
options.wave_size = input_info.wave_size;
}
DumpShaderOriginal(stage_name, options.shader_hash, params.code);
auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
if (translated.skip_dispatch) {
entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
@@ -365,7 +343,7 @@ struct PipelineCache::ProgramCache {
entry->second.specialization));
}
entry->second.permutations.push_back(CompilePermutation(
params, options, std::move(translated), entry->second.specialization, push_data_cursor));
stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor));
const auto& permutation = entry->second.permutations.back();
input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
permutation.program.bindings.AdvancePushData(push_data_cursor);
+20 -10
View File
@@ -575,12 +575,17 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
return state;
}
static bool DrawHasActivePixelShader(const CommandBuffer& buffer) {
const auto& ctx = buffer.GetRegisters();
const auto& sh_regs = ctx.GetShaderRegisters();
const bool has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
return buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
(has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
static uint32_t DrawColorOutputMask(const HW::Context& ctx) {
const auto& sh_regs = ctx.GetShaderRegisters();
const auto write_mask = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask;
uint32_t output_mask = 0;
for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
if (sh_regs.target_output_mode[slot] != 0 &&
render_target_mask_slot(write_mask, slot) != 0) {
output_mask |= 1u << slot;
}
}
return output_mask;
}
enum class CbColorMode : uint8_t {
@@ -847,7 +852,7 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
}
static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
DrawRenderState& state) {
uint32_t color_output_mask, DrawRenderState& state) {
auto& ctx = buffer.GetRegisters();
auto& sh_ctx = buffer.GetShaders();
@@ -861,7 +866,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
target_export_mapping {};
for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
const auto& rt = ctx.GetRenderTarget(slot);
if (rt.base.addr != 0 && render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0) {
if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) {
target_export_mapping[slot] =
TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type,
rt.info.channel_order)
@@ -880,8 +885,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
uint32_t render_target_slice_offset,
DrawRenderState& state) {
state.ps_active = DrawHasActivePixelShader(buffer);
RefreshShaders(buffer, draw, state);
const auto& shader_regs = buffer.GetRegisters().GetShaderRegisters();
const auto color_output_mask = DrawColorOutputMask(buffer.GetRegisters());
state.ps_active = buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
(color_output_mask != 0 ||
PixelShaderHasDepthOrCoverageSideEffects(shader_regs));
RefreshShaders(buffer, draw, color_output_mask, state);
uint32_t mrt_mask = 0;
if (state.ps_active) {
for (const auto& output: state.ps_input_info.stage.program->info.outputs) {
@@ -890,6 +899,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
}
}
}
mrt_mask &= color_output_mask;
if (draw.IsIndexed()) {
LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
}
@@ -15,7 +15,6 @@
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
#include <algorithm>
@@ -548,30 +547,40 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash);
auto cfg = CFG::BuildGraph(decoded);
auto native_cfg = CFG::BuildGraph(decoded);
CFG::Graph structured_cfg;
auto* selected_cfg = &native_cfg;
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
" loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
if (cfg.irreducible) {
LogDispatcherFallback(options, cfg, "build");
static_cast<uint64_t>(native_cfg.blocks.size()),
static_cast<uint64_t>(native_cfg.natural_loops.size()),
static_cast<uint64_t>(native_cfg.back_edges.size()), phase_ms());
if (native_cfg.irreducible) {
LogDispatcherFallback(options, native_cfg, "build");
} else {
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!CFG::Structurize(cfg)) {
LogDispatcherFallback(options, cfg, "structurize");
structured_cfg = CFG::Structurize(native_cfg);
if (structured_cfg.unsupported) {
native_cfg.unsupported = true;
native_cfg.failure_kind = structured_cfg.failure_kind;
native_cfg.failure_block = structured_cfg.failure_block;
native_cfg.unsupported_reason = structured_cfg.unsupported_reason;
LogDispatcherFallback(options, native_cfg, "structurize");
} else {
selected_cfg = &structured_cfg;
LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
static_cast<uint64_t>(cfg.blocks.size()));
static_cast<uint64_t>(selected_cfg->blocks.size()));
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
" loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(cfg.blocks.size()),
static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
static_cast<uint64_t>(selected_cfg->blocks.size()),
static_cast<uint64_t>(selected_cfg->natural_loops.size()), phase_ms());
}
const auto& cfg = *selected_cfg;
Frontend::EmbeddedFetchPlan embedded_fetch;
if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
@@ -614,15 +623,21 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
IR::EliminateDeadCode(ir.blocks);
}
LowerTessellationMemory(ir, options);
IR::BuildSrtPlan(ir);
IR::EliminateDeadCode(ir.blocks);
IR::TrackResources(ir);
std::string cfg_dump;
if (options.dump_ir) {
cfg_dump = CFG::GraphToString(cfg);
if (options.early_dump) {
LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options),
MakeIrDump(cfg_dump, ir).c_str());
}
}
IR::TrackResources(ir, decoded, native_cfg);
IR::EliminateDeadCode(ir.blocks);
TranslateResult result;
result.program = std::move(ir);
if (options.dump_ir) {
result.decoded_dump = std::move(decoded_dump);
result.cfg_dump = CFG::GraphToString(cfg);
result.cfg_dump = std::move(cfg_dump);
}
return result;
}
@@ -634,6 +649,35 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
const auto emit_begin = std::chrono::steady_clock::now();
auto& ir = translated.program;
IR::ApplyResourceSpecialization(ir, specialization);
// The resource plan owns host descriptor evaluation now. Keep only dependencies consumed
// by GPU memory operations; bound descriptor dwords must not retain shader instructions.
for (auto& inst: ir.value_storage) {
inst.Invalidate();
}
for (auto* block: ir.blocks) {
for (auto& inst: *block) {
const auto op = inst.GetOpcode();
uint32_t first = 0;
if (op == IR::ValueOpcode::GetBufferResource) {
if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) {
return ir.memory_info[use.user->Flags<IR::MemoryFlags>().index].kind ==
IR::ResourceKind::IndirectBuffer;
})) {
continue;
}
} else if (op == IR::ValueOpcode::GetImageResource) {
const auto resource = inst.Flags<uint32_t>();
first = resource < ir.info.images.size() &&
ir.info.images[resource].indirect_root == resource ? 1u : 0u;
} else if (op != IR::ValueOpcode::GetSamplerResource) {
continue;
}
for (size_t index = first; index < inst.NumArgs(); index++) {
inst.SetArg(index, IR::Value(0u));
}
}
}
ir.value_storage.clear();
IR::RemoveIdentities(ir.blocks);
IR::EliminateDeadCode(ir.blocks);
@@ -62,12 +62,7 @@ void ValidateNativeProgram(const IR::Program& program) {
Expect(Kind::BdaPagetable);
Expect(Kind::FaultBuffer);
}
const bool uses_flattened_runtime =
!program.srt_reads.empty() ||
std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
return image.indirect_search_iterations != 0u;
});
if (uses_flattened_runtime) {
if (IR::UsesFlattenedSrt(program)) {
Expect(Kind::FlattenedSrt);
}
if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) {
@@ -203,9 +203,7 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
const auto bits = state.builder.AllocateId();
const auto abs_bits = state.builder.AllocateId();
const auto sign_bits = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto subnormal = state.builder.AllocateId();
const auto flush = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
@@ -213,12 +211,9 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
ConstantU32(state, 0x7fffffffu));
state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
ConstantU32(state, 0x80000000u));
state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits,
ConstantU32(state, 0));
state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
ConstantU32(state, 0x00800000u));
state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal);
state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits);
state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits);
state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
return ret;
}
@@ -636,6 +636,29 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Arg(inst, 1));
}
uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) {
if (ctx.other_half == nullptr) return ctx.Arg(inst, 0);
// A native scalar branch makes one decision for both emulated wave halves.
if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst));
const auto kind = inst.Flags<CFG::BranchCondition>();
if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0);
const auto ballot = ctx.Ballot(inst.Arg(0));
const auto low = ctx.state.builder.AllocateId();
const auto high = ctx.state.builder.AllocateId();
const auto combined = ctx.state.builder.AllocateId();
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
const bool zero = kind == CFG::BranchCondition::ExecZero ||
kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero;
ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
TypeU32(ctx.state), combined, low, high);
ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual,
TypeBool(ctx.state), result, combined,
ConstantU32(ctx.state, zero ? ~0u : 0u));
return result;
}
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
return ctx.Ballot(predicate);
}
@@ -2,6 +2,8 @@
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include <bit>
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
// Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
// (IR::Value), and instructions whose metadata or operand loads must stay lazy.
@@ -112,18 +114,34 @@ EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t)
EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
template <spv::Op opcode>
uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
// SPIR-V's DenormFlushToZero mode does not require comparison operands to flush.
const bool flush = inst.Flags<IR::FPCompareFlags>().flush_input_denorms;
const auto operand = [&](size_t index) {
const auto value = inst.Arg(index);
if (flush && value.IsImmediate()) {
const auto bits = std::bit_cast<uint32_t>(value.F32Value());
return ConstantF32(ctx.state, (bits & 0x7fffffffu) < 0x00800000u
? bits & 0x80000000u : bits);
}
const auto id = ctx.Arg(inst, index);
return flush ? EmitFlushF32DenormToSignedZero(ctx.state, id) : id;
};
return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
}
inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
inline constexpr auto EmitFPOrdLessThan32 = EmitFloatCompare32<spv::OpFOrdLessThan>;
inline constexpr auto EmitFPUnordLessThan32 = EmitFloatCompare32<spv::OpFUnordLessThan>;
inline constexpr auto EmitFPOrdGreaterThan32 = EmitFloatCompare32<spv::OpFOrdGreaterThan>;
inline constexpr auto EmitFPUnordGreaterThan32 = EmitFloatCompare32<spv::OpFUnordGreaterThan>;
inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32<spv::OpFOrdLessThanEqual>;
inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32<spv::OpFUnordLessThanEqual>;
inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32<spv::OpFOrdGreaterThanEqual>;
inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32<spv::OpFUnordGreaterThanEqual>;
uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0);
inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
@@ -182,6 +200,7 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in
uint32_t EmitWqmU64(EmitterState& state, uint32_t value);
uint32_t EmitLaneId(EmitterState& state);
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
@@ -146,7 +146,12 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
ctx.Fail(inst, "has no address base pair");
return ConstantDeviceAddress(state, 0);
}
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
auto base_low = ctx.Arg(*handle, 0);
if (mem.kind == IR::ResourceKind::ScalarAddress) {
base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low,
ConstantU32(state, ~3u));
}
const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1));
address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
}
@@ -1099,16 +1104,27 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
if (ctx.half == 1) {
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
}
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, ctx.Memory(inst).offset));
const auto mem = ctx.Memory(inst);
const auto offset = mem.offset & 0xfffcu;
auto address = ConstantU32(state, offset);
uint32_t region_bounds = 0;
if (mem.kind == IR::ResourceKind::Gds) {
const auto m0 = ctx.Arg(inst, 0);
const auto base =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size =
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
address = Binary(state, spv::OpIAdd, TypeU32(state), base, address);
// The entire M0 region must fit the PS5's 48 KiB GDS partition.
region_bounds = AndCondition(
state, Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, offset + 3u), size),
Binary(state, spv::OpULessThanEqual, TypeBool(state),
Binary(state, spv::OpIAdd, TypeU32(state), base, size),
ConstantU32(state, 0xc000u)));
}
const auto raw_index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index);
const auto exec = ctx.Arg(inst, 1);
@@ -1127,19 +1143,17 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
: first;
const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds
? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
auto bounds = EmitMemoryElementInBounds(state, access, index);
if (mem.kind == IR::ResourceKind::Gds) {
bounds = AndCondition(state, bounds, region_bounds);
}
const auto condition = AndCondition(
state, is_first,
AndCondition(state,
state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
ConstantU32(state, 0))
: exec,
AndCondition(state, storage_bounds, m0_bounds)));
bounds));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state),
@@ -1173,10 +1187,11 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
if (mem.planning_only) return;
auto& state = ctx.state;
mem.kind = IR::ResourceKind::ScalarBuffer;
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, mem.offset));
const auto index =
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
ConstantU32(state, 2));
if (mem.offset >= 4u) {
index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
}
const auto access = PrepareMemoryResourceAccess(state, mem);
const auto element = EmitMemoryElementIndex(state, access, index);
const auto condition = EmitMemoryElementInBounds(state, access, element);
@@ -1198,6 +1213,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
value = LoadWideBuffer(ctx, inst, buffer_components);
else if (shared_components > 1u)
value = LoadWideShared(ctx, inst, shared_components);
else if (mem.kind == IR::ResourceKind::ScalarAddress)
value = LoadBdaDword(ctx, GuestAddress(ctx, inst, mem));
else if (address_info.access == IR::AddressAccess::Read &&
mem.kind != IR::ResourceKind::Scratch)
value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),
@@ -97,30 +97,6 @@ void EmitReturn(ValueEmitContext& ctx) {
ctx.state.builder.AddFunction(spv::OpReturn);
}
uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
// Scalar-instruction conditions already test the full wave's raw register values.
if (ctx.other_half == nullptr ||
info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
info.terminator.condition == CFG::BranchCondition::GotoVariable) {
return ctx.Def(info.condition);
}
const auto ballot = ctx.Ballot(info.condition);
const auto low = ctx.state.builder.AllocateId();
const auto high = ctx.state.builder.AllocateId();
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
const auto kind = info.terminator.condition;
const bool zero = kind == CFG::BranchCondition::ExecZero ||
kind == CFG::BranchCondition::VccZero ||
kind == CFG::BranchCondition::SccZero;
const auto combined =
EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u));
return result;
}
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
const IR::BlockInfo& info) {
const auto& program = ctx.state.program;
@@ -160,7 +136,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
EmitReturn(ctx);
return;
}
const auto condition = BranchCondition(ctx, info);
const auto condition = ctx.Def(info.condition);
emit_merge();
ctx.state.builder.AddFunction(spv::OpBranchConditional, condition,
ctx.Label(true_block), ctx.Label(false_block));
@@ -193,7 +169,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
const auto selected = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(
spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info),
spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition),
ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block));
return selected;
}
File diff suppressed because it is too large Load Diff
@@ -18,10 +18,17 @@ enum class BranchCondition {
ExecZero,
ExecNonZero,
ScalarInstruction,
GotoVariable,
Expression,
Unknown
};
struct ConditionExpression {
enum class Op { Constant, Variable, Native, Not, Or };
Op op;
uint32_t lhs;
uint32_t rhs = UINT32_MAX;
};
enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported };
enum class FailureKind {
@@ -48,8 +55,7 @@ struct Terminator {
std::vector<uint32_t> indirect_targets;
std::vector<uint32_t> indirect_selector_values;
std::vector<uint32_t> indirect_selector_targets;
uint32_t goto_variable = UINT32_MAX;
int32_t goto_value = -1;
uint32_t expression = UINT32_MAX;
bool loop_header = false;
};
@@ -59,10 +65,11 @@ struct BasicBlock {
uint32_t end_pc = 0;
uint32_t inst_begin = 0;
uint32_t inst_end = 0;
struct Assignment { uint32_t variable; uint32_t expression; };
std::vector<Assignment> assignments;
std::vector<uint32_t> predecessors;
std::vector<uint32_t> successors;
std::vector<uint32_t> dominators;
std::vector<uint32_t> post_dominators;
Terminator terminator;
};
@@ -73,12 +80,9 @@ struct BackEdge {
};
struct NaturalLoop {
uint32_t header = UINT32_MAX;
uint32_t latch = UINT32_MAX;
uint32_t merge = UINT32_MAX;
uint32_t continue_block = UINT32_MAX;
uint32_t header = UINT32_MAX;
uint32_t latch = UINT32_MAX;
std::vector<uint32_t> body_blocks;
std::vector<uint32_t> exit_blocks;
};
struct StronglyConnectedComponent {
@@ -89,6 +93,7 @@ struct StronglyConnectedComponent {
struct Graph {
std::vector<BasicBlock> blocks;
std::vector<ConditionExpression> expressions;
std::vector<BackEdge> back_edges;
std::vector<NaturalLoop> natural_loops;
std::vector<StronglyConnectedComponent> components;
@@ -105,14 +110,12 @@ struct Graph {
const BasicBlock* FindBlockByPc(uint32_t pc) const;
BasicBlock* FindBlockByPc(uint32_t pc);
bool Dominates(uint32_t dominator, uint32_t block) const;
bool PostDominates(uint32_t post_dominator, uint32_t block) const;
uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
};
Graph BuildGraph(const Decoder::Program& program);
// Commits structured control flow on success; preserves the original graph with
// failure diagnostics on failure. failure_block is an original block ID or UINT32_MAX.
bool Structurize(Graph& graph);
// Returns structured control flow or failure diagnostics without changing the native graph.
// On failure, failure_block is an original block ID or UINT32_MAX.
Graph Structurize(const Graph& graph);
std::string BranchConditionToString(BranchCondition condition);
std::string FailureKindToString(FailureKind kind);
std::string GraphToString(const Graph& graph);
@@ -44,7 +44,8 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc
half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
const auto rhs =
half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
}
void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) {
@@ -235,6 +235,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
return index == 0u ? decoded.src0 : MakeImmediate(decoded.offset & 0xffffu);
case Decoder::Opcode::DS_CONSUME:
case Decoder::Opcode::DS_APPEND:
return decoded.gds ? MakeM0Operand() : MakeImmediate(0);
case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
case Decoder::Opcode::DS_WRITE_ADDTID_B32:
return index == 0u ? decoded.src1 : MakeM0Operand();
@@ -845,8 +846,7 @@ void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode
void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
const auto memory = MemoryInfoFromDecoded(inst);
WriteOperand(inst.dst, ir.Emit(opcode,
{ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(),
ir.GetExecHi()},
{ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
AddMemoryInfo(memory, inst.pc)));
}
@@ -317,6 +317,7 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
case Decoder::OperandKind::Sgpr: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
ir.SetScalarReg(reg, value);
program.scalar_writes.push_back({current_pc, reg});
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
@@ -699,11 +700,13 @@ std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
program.scalar_writes.push_back({current_pc, reg});
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
if ((write_64 || program.wave_size == 64u) &&
IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
program.scalar_writes.push_back({current_pc, high});
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
@@ -746,55 +749,61 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
}
void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) {
if (source.terminator.goto_value >= 0) {
if (source.terminator.goto_variable == UINT32_MAX) {
EXIT("block %u sets an invalid goto variable", source.id);
void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
IR::BlockInfo& info) {
const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 {
IR::U1 condition;
switch (kind) {
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
case CFG::BranchCondition::ScalarInstruction:
EXIT_IF(instruction_branch_condition.IsEmpty());
condition = instruction_branch_condition;
break;
default: EXIT("block %u has an invalid native branch condition", source.id);
}
ir.SetGotoVariable(source.terminator.goto_variable,
IR::U1(IR::Value(source.terminator.goto_value != 0)));
return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind));
};
const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 {
const auto& value = graph.expressions.at(index);
switch (value.op) {
case CFG::ConditionExpression::Op::Constant: return IR::U1(IR::Value(value.lhs != 0));
case CFG::ConditionExpression::Op::Variable: return ir.GetGotoVariable(value.lhs);
case CFG::ConditionExpression::Op::Native:
return native_condition(static_cast<CFG::BranchCondition>(value.lhs));
case CFG::ConditionExpression::Op::Not: return ir.LogicalNot(self(self, value.lhs));
case CFG::ConditionExpression::Op::Or:
return ir.LogicalOr(self(self, value.lhs), self(self, value.rhs));
}
EXIT("invalid CFG condition expression");
};
for (const auto& assignment: source.assignments) {
ir.SetGotoVariable(assignment.variable, expression(expression, assignment.expression));
}
if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) {
if (source.terminator.indirect_selector_code != UINT32_MAX) {
info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code);
} else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) {
info.indirect_target =
ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr));
const auto& term = source.terminator;
if (term.kind == CFG::TerminatorKind::IndirectBranch) {
if (term.indirect_selector_code != UINT32_MAX) {
info.indirect_target = ReadScalarCode(term.indirect_selector_code);
} else if (term.indirect_pc_sgpr != UINT32_MAX) {
info.indirect_target = ir.GetScalarReg(static_cast<IR::ScalarReg>(term.indirect_pc_sgpr));
} else {
EXIT("block %u has no indirect branch selector", source.id);
}
ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target});
return;
}
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
return;
if (term.kind == CFG::TerminatorKind::ConditionalBranch) {
const auto condition = term.expression != UINT32_MAX
? expression(expression, term.expression)
: native_condition(term.condition);
info.condition = condition;
ir.Emit(IR::ValueOpcode::Reference, {condition});
}
// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
// invocations leave the region without reconstructing a host-subgroup mask.
IR::U1 condition;
switch (source.terminator.condition) {
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
case CFG::BranchCondition::ScalarInstruction:
EXIT_IF(instruction_branch_condition.IsEmpty());
condition = instruction_branch_condition;
break;
case CFG::BranchCondition::GotoVariable:
if (source.terminator.goto_variable == UINT32_MAX) {
EXIT("block %u reads an invalid goto variable", source.id);
}
condition = ir.GetGotoVariable(source.terminator.goto_variable);
break;
case CFG::BranchCondition::Unknown:
EXIT("block %u has an unknown branch condition", source.id);
}
info.condition = condition;
ir.Emit(IR::ValueOpcode::Reference, {condition});
}
namespace {
@@ -1272,9 +1281,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
builtin(IR::StageInputKind::InstanceIndex));
}
}
const bool flush_f32_inputs = options.stage == ShaderType::Compute &&
(options.input_info.compute->float_mode & 0x10u) == 0;
for (const auto& cfg_block: cfg.blocks) {
const auto typed_index = block_indices.at(cfg_block.id);
Translator translator(result, result.blocks[typed_index], vector_limit);
Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs);
for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
const auto& instruction = decoded.instructions[index];
if (IsCodeTableLoad(cfg, instruction.pc)) {
@@ -1297,7 +1308,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
}
translator.TranslateInstruction(instruction);
}
translator.AddBranchCondition(cfg_block, result.block_info[typed_index]);
translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]);
}
IR::ValidateProgram(result, false);
return result;
@@ -9,13 +9,15 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
class Translator {
public:
Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit)
: program(program), ir(block), current_vector_limit(vector_limit) {}
Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit,
bool flush_f32_inputs)
: program(program), ir(block), current_vector_limit(vector_limit),
flush_f32_inputs(flush_f32_inputs) {}
void TranslateInstruction(const Decoder::Instruction& inst);
void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
uint32_t component_count, const ShaderBufferResource& resource);
void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info);
void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info);
private:
const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index);
@@ -261,6 +263,7 @@ private:
Decoder::Opcode current_opcode = Decoder::Opcode::UNKNOWN;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
bool flush_f32_inputs;
};
} // namespace Libs::Graphics::ShaderRecompiler::Frontend
@@ -81,7 +81,8 @@ bool EquivalentValue(const ResourcePlan& program, Value left, Value right,
program.memory_info[li] != program.memory_info[ri]) {
return false;
}
} else if (lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
} else if (lhs->GetOpcode() != ValueOpcode::ReadConst &&
lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
return false;
}
for (size_t index = 0; index < lhs->NumArgs(); index++) {
+3 -1
View File
@@ -147,6 +147,7 @@ struct SamplerResource {
uint32_t first_use_pc = 0;
bool force_point_filtering = false;
bool depth_compare = false;
bool integer_border = false;
bool operator==(const SamplerResource& other) const = default;
};
@@ -582,10 +583,11 @@ struct Program: ResourcePlan {
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
std::string fallback_reason;
std::vector<BlockInfo> block_info;
struct ScalarWrite { uint32_t pc; ScalarReg reg; };
std::vector<ScalarWrite> scalar_writes;
// Typed memory and export instructions reference shader-local metadata by dense index.
// Decoder-only details (such as NSA register numbers) have already become IR operands.
std::vector<ExportInfo> export_info;
std::vector<Value> dynamic_reads;
bool shader_info_complete = false;
BindingLayout bindings;
bool binding_layout_complete = false;
@@ -52,6 +52,10 @@ struct PermlaneFlags {
static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t));
static_assert(std::is_trivially_copyable_v<PermlaneFlags>);
struct FPCompareFlags {
bool flush_input_denorms = false;
};
struct MemoryFlags {
uint32_t index = 0;
uint32_t pc = 0;
@@ -4,6 +4,7 @@ VALUE_OPCODE(Identity, Opaque, Opaque)
VALUE_OPCODE(Void, Void)
VALUE_OPCODE(Reference, Void, U1)
VALUE_OPCODE(ReferenceU32, Void, U32)
VALUE_OPCODE(ConditionRef, U1, U1)
// Translation and register-state pseudo operations.
VALUE_OPCODE(GetUserData, U32, ScalarReg)
@@ -274,8 +275,8 @@ VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32)
VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32)
VALUE_OPCODE(DataAppend, U32, U32, U1)
VALUE_OPCODE(DataConsume, U32, U32, U1)
VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
@@ -85,6 +85,16 @@ bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffe
return uses_gds;
}
bool UsesFlattenedSrt(const Program& program) {
return std::ranges::any_of(program.blocks, [](const Block* block) {
return std::ranges::any_of(*block, [](const Inst& inst) {
return inst.GetOpcode() == ValueOpcode::ReadConst;
});
}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) {
return image.indirect_search_iterations != 0u;
});
}
void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
if (!program.shader_info_complete || program.binding_layout_complete) {
EXIT("shader binding layout failed: %s", !program.shader_info_complete
@@ -145,12 +155,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
AddBinding(next, DescriptorBindingKind::BdaPagetable);
AddBinding(next, DescriptorBindingKind::FaultBuffer);
}
const bool uses_flattened_runtime =
!program.srt_reads.empty() ||
std::ranges::any_of(program.info.images, [](const ImageResource& image) {
return image.indirect_search_iterations != 0u;
});
if (uses_flattened_runtime) {
if (UsesFlattenedSrt(program)) {
AddBinding(next, DescriptorBindingKind::FlattenedSrt);
}
@@ -9,6 +9,7 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0);
// Collect live direct buffer indices and report whether the shader accesses GDS.
bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers);
bool UsesFlattenedSrt(const Program& program);
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
@@ -630,6 +630,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
}
}
return;
case ValueOpcode::ConditionRef: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U1)) Replace(inst, value);
return;
}
case ValueOpcode::LogicalNot: {
const auto value = Arg(inst, 0);
if (IsImmediate(value, Type::U1)) {
@@ -107,14 +107,16 @@ Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) {
: Prospero::BufferFormat::kInvalid;
}
bool RequiresPointSampler(const ImageResource& image) {
return image.numeric_class == Prospero::TextureNumericClass::Sint ||
image.conversion_format != Prospero::BufferFormat::kInvalid;
}
enum class SamplerClass : uint8_t { Float, Integer, PointInteger };
bool RequiresPointSampler(const ResourceSpecialization::Image& image) {
return image.numeric_class == Prospero::TextureNumericClass::Sint ||
image.conversion_format != Prospero::BufferFormat::kInvalid;
template <typename Image>
SamplerClass ClassifySampler(const Image& image) {
if (image.numeric_class == Prospero::TextureNumericClass::Sint ||
image.conversion_format != Prospero::BufferFormat::kInvalid) {
return SamplerClass::PointInteger;
}
return image.numeric_class == Prospero::TextureNumericClass::Uint ? SamplerClass::Integer
: SamplerClass::Float;
}
bool DescriptorIsCube(const DescriptorValue& descriptor) {
@@ -223,9 +225,9 @@ void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* sou
}
}
bool ReadScalarTable(uint64_t base, uint64_t size, uint32_t dynamic_offset,
bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
const SrtRuntime& runtime, std::span<uint32_t> words) {
const auto offset = static_cast<uint64_t>(dynamic_offset) & ~uint64_t {3};
const auto offset = dynamic_offset & ~uint64_t {3};
const auto count = std::min<uint64_t>(words.size(), offset < size ? (size - offset) / 4u : 0u);
std::ranges::fill(words.subspan(count), 0u);
if (count == 0u) {
@@ -304,26 +306,24 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
material.Stride() != indirect.selector_stride) {
return false;
}
// Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds.
const auto step = std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u);
const auto residue = static_cast<uint64_t>(indirect.selector_offset) % step;
const auto limit = std::min<uint64_t>(UINT32_MAX, material.GetSize() + 3u);
const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
// The first aligned offset includes the immediate added after shader U32 arithmetic.
const auto step = std::max<uint64_t>(4u,
std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
const uint64_t first = indirect.selector_offset;
const auto size = material.GetSize();
const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
if (probe_count > MaxIndirectImageProbes) {
return false;
}
keys.reserve(static_cast<size_t>(probe_count) + 1u);
keys.push_back(0u);
for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
uint32_t key = 0;
if (!ReadScalarTable(material.Base48(), material.GetSize(),
static_cast<uint32_t>(offset), runtime, {&key, 1})) {
if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
return false;
}
keys.push_back(key);
if (limit - offset < step) {
break;
}
}
std::ranges::sort(keys);
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
@@ -380,8 +380,13 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
} // namespace
struct SamplerPlan {
std::array<uint32_t, ShaderInfo::MaxSamplers> point_sampler {};
uint32_t sampler_count = 0;
struct Binding {
uint32_t source;
SamplerClass type;
};
std::array<std::array<uint32_t, 3>, ShaderInfo::MaxSamplers> mapping;
std::array<Binding, ShaderInfo::MaxSamplers> bindings;
uint32_t sampler_count = 0;
};
struct ImageRemap {
@@ -595,11 +600,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) {
return SpecializationFail("specialized sampler layout exceeds its resource limit");
}
for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
const auto target = sampler_plan.point_sampler[index];
if (target != UINT32_MAX && target >= program.info.samplers.size()) {
snapshot.samplers.push_back(snapshot.samplers[index]);
}
for (uint32_t index = static_cast<uint32_t>(program.info.samplers.size());
index < sampler_plan.sampler_count; index++) {
snapshot.samplers.push_back(snapshot.samplers[sampler_plan.bindings[index].source]);
}
ImageRemap(specialization).Apply(snapshot.images);
return true;
@@ -607,29 +610,29 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
template <typename Images>
bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) {
if (base.samplers.size() > plan.point_sampler.size()) {
if (base.samplers.size() > plan.mapping.size()) {
return false;
}
std::array<uint8_t, ShaderInfo::MaxSamplers> usage {};
plan.point_sampler.fill(UINT32_MAX);
plan.sampler_count = static_cast<uint32_t>(base.samplers.size());
for (const auto& pair: base.sampled_pairs) {
if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) {
return false;
}
usage[pair.sampler] |= RequiresPointSampler(images[pair.image]) ? 2u : 1u;
usage[pair.sampler] |= 1u << static_cast<uint32_t>(ClassifySampler(images[pair.image]));
}
for (uint32_t index = 0; index < base.samplers.size(); index++) {
if ((usage[index] & 2u) == 0u) {
continue;
}
if ((usage[index] & 1u) == 0u) {
plan.point_sampler[index] = index;
} else {
if (plan.sampler_count >= ShaderInfo::MaxSamplers) {
return false;
}
plan.point_sampler[index] = plan.sampler_count++;
auto& mapping = plan.mapping[index];
mapping.fill(UINT32_MAX);
const auto classes = usage[index] == 0u ? 1u : usage[index];
bool first = true;
for (uint32_t type = 0; type < mapping.size(); type++) {
if ((classes & (1u << type)) == 0u) continue;
const auto target = first ? index : plan.sampler_count++;
if (target >= ShaderInfo::MaxSamplers) return false;
mapping[type] = target;
plan.bindings[target] = {index, static_cast<SamplerClass>(type)};
first = false;
}
}
return true;
@@ -1105,25 +1108,18 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
auto samplers = program.info.samplers;
auto sampled_pairs = program.info.sampled_pairs;
samplers.reserve(sampler_plan.sampler_count);
for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
const auto target = sampler_plan.point_sampler[index];
if (target == UINT32_MAX) {
continue;
}
if (target == index) {
samplers[index].force_point_filtering = true;
} else {
EXIT_IF(target != samplers.size());
auto sampler = samplers[index];
sampler.force_point_filtering = true;
samplers.push_back(std::move(sampler));
for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) {
const auto& binding = sampler_plan.bindings[index];
if (index >= program.info.samplers.size()) {
samplers.push_back(program.info.samplers[binding.source]);
}
samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger;
samplers[index].integer_border = binding.type != SamplerClass::Float;
}
for (auto& pair: sampled_pairs) {
if (RequiresPointSampler(images[pair.image])) {
EXIT_IF(sampler_plan.point_sampler[pair.sampler] == UINT32_MAX);
pair.sampler = sampler_plan.point_sampler[pair.sampler];
}
const auto type = static_cast<uint32_t>(ClassifySampler(images[pair.image]));
pair.sampler = sampler_plan.mapping[pair.sampler][type];
EXIT_IF(pair.sampler == UINT32_MAX);
samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare;
}
@@ -1201,10 +1197,11 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
inst.ReplaceUsesWith(Value(&*result));
continue;
}
if (image_opcode.needs_sampler && RequiresPointSampler(image) &&
if (image_opcode.needs_sampler &&
memory.sampler < program.info.samplers.size()) {
EXIT_IF(sampler_plan.point_sampler[memory.sampler] == UINT32_MAX);
memory.sampler = sampler_plan.point_sampler[memory.sampler];
const auto type = static_cast<uint32_t>(ClassifySampler(image));
memory.sampler = sampler_plan.mapping[memory.sampler][type];
EXIT_IF(memory.sampler == UINT32_MAX);
}
EXIT_IF(image.indirect_root == memory.resource &&
inst.GetOpcode() != ValueOpcode::ImageSampleRaw);
File diff suppressed because it is too large Load Diff
@@ -5,9 +5,8 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
// descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
void TrackResources(Program& program);
// Resolves native descriptor sources, plans their scalar reads, and assigns dense resource bindings.
void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg);
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -7,7 +7,6 @@
#include <bit>
#include <cmath>
#include <cstring>
#include <fmt/format.h>
#include <unordered_set>
namespace Libs::Graphics::ShaderRecompiler::IR {
@@ -23,21 +22,6 @@ namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
std::string Diagnostic(const ResourcePlan& program, uint32_t pc, const std::string& message) {
return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
StageName(program.stage), pc, message);
}
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
if (base > AddressMask) {
return false;
@@ -89,6 +73,7 @@ bool IsRuntimeSelect(ValueOpcode op) {
bool IsRuntimeUniformOp(ValueOpcode op) {
switch (op) {
case ValueOpcode::ConditionRef:
case ValueOpcode::BitCastU32F32:
case ValueOpcode::BitCastF32U32:
case ValueOpcode::ConvertU32F32:
@@ -312,151 +297,6 @@ private:
std::unordered_set<const Inst*> m_validated_dependencies;
};
class PlanBuilder {
public:
explicit PlanBuilder(Program& program): m_program(program) {}
void Run() {
m_program.srt_reads.clear();
m_program.dynamic_reads.clear();
for (auto* block: m_program.blocks) {
for (auto& inst: *block) {
const auto op = inst.GetOpcode();
if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
const auto flags = inst.Flags<MemoryFlags>();
if (flags.index < m_program.memory_info.size()) {
const auto kind = m_program.memory_info[flags.index].kind;
const bool crosswired = (op == ValueOpcode::LoadAddressU32 &&
kind == ResourceKind::ScalarBuffer) ||
(op == ValueOpcode::ReadConstBuffer &&
kind == ResourceKind::ScalarAddress);
if (crosswired) {
Fail(flags.pc,
fmt::format("{} has incompatible scalar memory metadata",
ValueOpcodeName(op)));
}
}
}
if (IsDescriptorHandle(inst.GetOpcode())) {
for (size_t index = 0; index < inst.NumArgs(); index++) {
Collect(inst.Arg(index), 0);
}
}
}
}
for (auto* block: m_program.blocks) {
for (auto& inst: *block) {
if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) &&
inst.Arg(1).Resolve().IsImmediate() &&
ValidateRuntimeValue(m_program, Value(&inst))) {
Collect(Value(&inst), inst.Flags<MemoryFlags>().pc);
}
}
}
PatchReads();
}
private:
struct Patch {
Inst* inst = nullptr;
uint32_t slot = 0;
bool keep = false;
};
[[noreturn]] void Fail(uint32_t pc, const std::string& message) const {
const auto diagnostic = Diagnostic(m_program, pc, message);
EXIT("shader SRT planning failed: %s", diagnostic.c_str());
std::abort();
}
void Collect(Value value, uint32_t use_pc) {
value = value.Resolve();
if (value.IsImmediate()) {
return;
}
auto* inst = value.TryInstruction();
if (inst == nullptr) {
Fail(use_pc, "invalid typed planning value");
}
const auto cycle = std::ranges::find(m_visiting, inst);
if (cycle != m_visiting.end()) {
const auto contains_phi = std::any_of(cycle, m_visiting.end(), [](const Inst* value) {
return value->GetOpcode() == ValueOpcode::Phi;
});
if (contains_phi) {
return;
}
Fail(use_pc, fmt::format("cyclic typed planning value {} without a phi",
ValueOpcodeName(inst->GetOpcode())));
}
if (std::ranges::find(m_visited, inst) != m_visited.end()) {
return;
}
m_visiting.push_back(inst);
for (size_t index = 0; index < inst->NumArgs(); index++) {
Collect(inst->Arg(index), use_pc);
}
m_visiting.pop_back();
m_visited.push_back(inst);
if (!IsRawRead(m_program, *inst)) {
return;
}
const auto offset = inst->Arg(1).Resolve();
if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
if (std::ranges::find(m_program.dynamic_reads, value) ==
m_program.dynamic_reads.end()) {
m_program.dynamic_reads.push_back(value);
}
return;
}
for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) {
if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) {
m_patches.push_back({inst, slot, false});
return;
}
}
const auto slot = static_cast<uint32_t>(m_program.srt_reads.size());
m_program.srt_reads.push_back({value, slot});
m_patches.push_back({inst, slot, true});
}
void PatchReads() {
for (const auto& patch: m_patches) {
auto* block = patch.inst->Parent();
auto& list = block->Instructions();
auto where =
std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; });
const auto resource =
Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource));
const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst,
{resource, Value(patch.slot)}));
const auto uses = patch.inst->Uses();
for (const auto& use: uses) {
use.user->SetArg(use.operand, flat);
}
for (auto& info: m_program.block_info) {
if (info.condition.Resolve() == Value(patch.inst)) {
info.condition = flat;
}
if (info.indirect_target.Resolve() == Value(patch.inst)) {
info.indirect_target = flat;
}
}
if (patch.keep) {
const auto memory = patch.inst->Flags<MemoryFlags>().index;
if (memory < m_program.memory_info.size()) {
m_program.memory_info[memory].planning_only = true;
}
block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
}
}
}
Program& m_program;
std::vector<Inst*> m_visiting;
std::vector<Inst*> m_visited;
std::vector<Patch> m_patches;
};
} // namespace
@@ -616,16 +456,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
return false;
}
const auto byte_offset =
static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
const auto aligned = byte_offset & ~uint64_t {3};
(static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
const auto stride = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
const auto size = stride == 0u
? static_cast<uint64_t>(static_cast<uint32_t>(records))
: static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
if (aligned > size || size - aligned < sizeof(uint32_t)) {
if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) {
return false;
}
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
address = (base & ~uint64_t {3}) + byte_offset;
} else {
const auto relative = (immediate & ~int64_t {3}) +
static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
@@ -780,14 +619,18 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
}
return false;
case ValueOpcode::FPOrdLessThanEqual32:
if (binary()) {
result = Float32(a) <= Float32(b);
return true;
}
return false;
case ValueOpcode::FPOrdGreaterThanEqual32:
if (binary()) {
result = Float32(a) >= Float32(b);
const auto operand = [&](uint64_t bits) {
if (inst.Flags<FPCompareFlags>().flush_input_denorms &&
(bits & 0x7fffffffu) < 0x00800000u) {
bits &= 0x80000000u;
}
return Float32(bits);
};
result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32
? operand(a) <= operand(b)
: operand(a) >= operand(b);
return true;
}
return false;
@@ -970,6 +813,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
return true;
}
return false;
case ValueOpcode::ConditionRef: return Arg(inst, 0, result);
case ValueOpcode::LogicalNot:
if (Arg(inst, 0, a)) {
result = a == 0u;
@@ -1061,13 +905,5 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValue
return RuntimeValidator(program, type).Run(value);
}
void BuildSrtPlan(Program& program) {
if (program.resource_tracking_complete) {
EXIT("shader SRT planning failed: cannot rebuild SRT after resource tracking");
}
program.srt_plan_complete = false;
PlanBuilder(program).Run();
program.srt_plan_complete = true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
@@ -21,9 +21,6 @@ struct SrtRuntime {
enum class RuntimeValueType { Any, Integer };
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
// dynamic offsets remain explicit and are never assigned a fake slot.
void BuildSrtPlan(Program& program);
bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
RuntimeValueType type = RuntimeValueType::Any);
// Uses the strict reader for values that affect shader specialization.
+444 -187
View File
@@ -1,6 +1,7 @@
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
@@ -126,8 +127,28 @@ struct Fixture {
}
void PlanAndTrack() {
BuildSrtPlan(program);
TrackResources(program);
for (size_t index = 0; index < program.block_info.size(); ++index) {
const auto condition = program.block_info[index].condition;
if (!condition.IsEmpty())
Emit(ValueOpcode::Reference, {condition}, 0, program.blocks[index]);
}
for (auto *target : program.blocks) {
for (auto &inst : *target) {
if (inst.HasUses() || inst.MayHaveSideEffects() ||
(BufferAccessOf(inst.GetOpcode()) == BufferAccess::None &&
AddressOpcodeInfoOf(inst.GetOpcode()).access == AddressAccess::None &&
ImageOpcodeInfoOf(inst.GetOpcode()).access == ImageAccess::None))
continue;
auto value = Value(&inst);
if (value.GetType() == Type::U32x4)
value = Emit(ValueOpcode::CompositeExtractU32x4, {value, Value(0u)}, 0, target);
if (value.GetType() == Type::U8)
value = Emit(ValueOpcode::ConvertU32U8, {value}, 0, target);
Check(value.GetType() == Type::U32, "unhandled memory result type in fixture");
Emit(ValueOpcode::ReferenceU32, {value}, 0, target);
}
}
TrackResources(program, {}, {});
}
};
@@ -185,8 +206,9 @@ bool ReadLinearTestMemory(void *userdata, uint64_t address, std::span<uint32_t>
}
std::unique_ptr<Fixture>
MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
bool memory_backed_material = false) {
MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 4,
bool memory_backed_material = false, uint32_t member_offset = 0,
uint32_t material_stride = 224) {
auto fixture = std::make_unique<Fixture>();
std::array<Value, 4> material_words;
std::array<Value, 4> heap_words;
@@ -228,9 +250,9 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0,
const auto selector = fixture->Emit(ValueOpcode::ReadFirstLane,
{invocation, Value(true)});
const auto record =
fixture->Emit(ValueOpcode::IMul32, {selector, Value(224u)});
const auto member = fixture->Emit(ValueOpcode::IAdd32, {record, Value(4u)});
fixture->Emit(ValueOpcode::ReferenceU32, {record});
fixture->Emit(ValueOpcode::IMul32, {selector, Value(material_stride)});
const auto member = member_offset == 0 ? record :
fixture->Emit(ValueOpcode::IAdd32, {record, Value(member_offset)});
fixture->Emit(ValueOpcode::ReferenceU32, {member});
MemoryInfo material_scalar;
material_scalar.kind = ResourceKind::ScalarBuffer;
@@ -277,7 +299,9 @@ void TestInvariantIndirectImageMaterialization() {
Check(fixture->program.info.buffers.size() == 1 &&
fixture->program.info.images.size() == 1 &&
fixture->program.dynamic_reads.size() == 1,
std::ranges::any_of(*fixture->block, [](const Inst &inst) {
return inst.GetOpcode() == ValueOpcode::ReadConstBuffer;
}),
"indirect image key was not retained as a scalar-buffer read");
const auto source = fixture->program.info.images[0].source;
Check(source < fixture->program.descriptor_sources.size() &&
@@ -474,7 +498,7 @@ void TestInvariantIndirectImageMaterialization() {
rebound_specialization != collapsed_specialization,
"larger indirect candidate topology reused the old specialization");
auto memory_backed = MakeIndirectImageFixture(false, 0u, true);
auto memory_backed = MakeIndirectImageFixture(false, 4u, true);
memory_backed->PlanAndTrack();
auto memory_backed_plan = ExtractResourcePlan(memory_backed->program);
EliminateDeadCode(memory_backed->program.blocks);
@@ -519,28 +543,50 @@ void TestInvariantIndirectImageMaterialization() {
"a same-shape refresh discarded the runtime output storage");
auto malformed = MakeIndirectImageFixture(true);
BuildSrtPlan(malformed->program);
CheckFatal([&] { TrackResources(malformed->program); }, "not a valid runtime value",
CheckFatal([&] { malformed->PlanAndTrack(); }, "not a valid runtime value",
"malformed indirect image pattern was accepted");
Check(!malformed->program.resource_tracking_complete &&
malformed->program.info.images.empty() &&
malformed->program.descriptor_sources.empty(),
"malformed indirect image pattern was partially accepted");
auto wrapped_immediate = MakeIndirectImageFixture(false, 4u);
BuildSrtPlan(wrapped_immediate->program);
CheckFatal([&] { TrackResources(wrapped_immediate->program); },
auto negative_immediate = MakeIndirectImageFixture(false, 0xfffffffcu);
CheckFatal([&] { negative_immediate->PlanAndTrack(); },
"not a valid runtime value",
"wrapped scalar immediate entered the invariant image proof");
Check(!wrapped_immediate->program.resource_tracking_complete,
"wrapped scalar immediate entered the invariant image proof");
"negative scalar immediate entered the indirect image proof");
for (const auto [immediate, member, first, stride] :
{std::array{0u, 4u, 4u, 224u}, std::array{4u, 4u, 8u, 224u},
std::array{36u, 0u, 36u, 224u}, std::array{1u, 3u, 0u, 224u},
std::array{1u, 3u, 0u, 1u}, std::array{1u, 3u, 0u, 2u}}) {
auto split_offset = MakeIndirectImageFixture(false, immediate, false, member, stride);
split_offset->PlanAndTrack();
const auto split_plan = ExtractResourcePlan(split_offset->program);
LinearTestMemory split_memory;
std::copy(image_descriptor.begin(), image_descriptor.end(),
split_memory.words.begin() + 0x1000u / 4u);
split_memory.fail_address = first == 36u ? 0x1004u : UINT64_MAX;
split_memory.watched_address = split_memory.base + first;
user_data[1] = stride << 16u;
user_data[2] = stride < 4u ? 8u / stride : 1u;
SrtRuntime split_runtime{.user_data = user_data,
.userdata = &split_memory,
.read_specialization_memory = ReadLinearTestMemory};
Check(MaterializeResources(split_plan, split_runtime, snapshot, specialization) &&
snapshot.images.size() == 1 &&
snapshot.images[0].dwords == image_descriptor &&
split_memory.watched_reads == 1u,
"indirect scalar offsets did not align and bound their components independently");
}
}
void TestGuardedDirectImageTable() {
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
enum class Guard { Nonzero, Zero, Unrelated, Bypass };
enum class Guard { Nonzero, SccNonZero, Plain, Zero, Unrelated, Bypass, ExecZero, VccZero };
const auto make_plan = [](Guard guard) {
Fixture fixture(ShaderType::Pixel);
fixture.program.wave_size = 64u;
auto *entry = fixture.block;
auto *middle = fixture.AddBlock();
auto *before_sample = fixture.AddBlock();
@@ -557,8 +603,19 @@ void TestGuardedDirectImageTable() {
const auto nonzero = fixture.Emit(
ValueOpcode::INotEqual32,
{Value(0u), guard == Guard::Unrelated ? fixture.UserData(2) : mask});
fixture.program.block_info[0].condition =
fixture.Emit(ValueOpcode::LogicalNot, {nonzero});
const auto kind = guard == Guard::ExecZero ? CFG::BranchCondition::ExecZero
: guard == Guard::VccZero ? CFG::BranchCondition::VccZero
: CFG::BranchCondition::SccZero;
auto condition = nonzero;
if (guard == Guard::SccNonZero) {
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
CFG::BranchCondition::SccNonZero);
}
condition = fixture.Emit(ValueOpcode::LogicalNot, {condition});
if (guard != Guard::Plain && guard != Guard::SccNonZero) {
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition}, kind);
}
fixture.program.block_info[0].condition = condition;
fixture.program.block_info[0].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = guard == Guard::Zero ? 1u : 4u,
@@ -622,7 +679,10 @@ void TestGuardedDirectImageTable() {
};
auto plan = make_plan(Guard::Nonzero);
for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass}) {
make_plan(Guard::SccNonZero);
make_plan(Guard::Plain);
for (const auto guard : {Guard::Zero, Guard::Unrelated, Guard::Bypass,
Guard::ExecZero, Guard::VccZero}) {
CheckFatal([&] { make_plan(guard); }, "not a valid runtime value",
"direct table accepted a selector without a dominating nonzero guard");
}
@@ -703,55 +763,183 @@ void TestGuardedDirectImageTable() {
void TestBoundedComputeImageLoop() {
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
enum class Variant {
Bounded, WrongGuard, EntryBypass, ExitBypass, GuardBlock, WrongStep
Bounded, Plain, Nonzero, TrueEdge, WrongGuard, EntryBypass, ExitBypass, GuardBlock,
WrongStep, DivergentBound, DivergentKey, Disjunction, WrongPolarity,
IncrementBypass, PreviousBound, Masked, MaskedWrongGuard, MaskResurrection, StatusOverwrite,
GuardedDiamond, GuardedDiamondBypass
};
const auto make_plan = [](Variant variant) {
Fixture fixture;
const bool diamond = variant >= Variant::GuardedDiamond;
Fixture fixture(diamond ? ShaderType::Vertex : ShaderType::Compute);
fixture.program.wave_size = 64u;
const bool masked = variant >= Variant::Masked;
auto *entry = fixture.block;
auto *header = fixture.AddBlock();
auto *body = fixture.AddBlock();
auto *latch = fixture.AddBlock();
auto *exit = fixture.AddBlock();
auto *compare = masked ? fixture.AddBlock() : header;
auto *guard = masked ? fixture.AddBlock() : header;
auto *increment = masked ? fixture.AddBlock() : latch;
auto *final_exit = variant == Variant::PreviousBound ? fixture.AddBlock() : exit;
entry->AddBranch(header);
header->AddBranch(exit);
header->AddBranch(body);
body->AddBranch(latch);
latch->AddBranch(header);
if (variant == Variant::EntryBypass) entry->AddBranch(body);
if (variant == Variant::ExitBypass) exit->AddBranch(body);
fixture.program.block_info[0].terminator = {
.kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
: CFG::TerminatorKind::Branch,
.true_block = 1u, .false_block = 2u};
fixture.program.block_info[1].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 4u, .false_block = 2u};
fixture.program.block_info[2].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 3u};
fixture.program.block_info[3].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 1u};
fixture.program.block_info[4].terminator = {
.kind = variant == Variant::ExitBypass ? CFG::TerminatorKind::Branch
: CFG::TerminatorKind::Return,
.true_block = 2u};
if (!masked) {
header->AddBranch(exit);
header->AddBranch(body);
body->AddBranch(latch);
latch->AddBranch(header);
if (variant == Variant::PreviousBound) latch->AddBranch(final_exit);
if (variant == Variant::EntryBypass) entry->AddBranch(body);
if (variant == Variant::ExitBypass) exit->AddBranch(body);
if (variant == Variant::IncrementBypass || variant == Variant::PreviousBound)
exit->AddBranch(latch);
fixture.program.block_info[0].terminator = {
.kind = variant == Variant::EntryBypass ? CFG::TerminatorKind::ConditionalBranch
: CFG::TerminatorKind::Branch,
.true_block = 1u, .false_block = 2u};
fixture.program.block_info[1].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = variant == Variant::TrueEdge ? 2u : 4u,
.false_block = variant == Variant::TrueEdge ? 4u : 2u};
fixture.program.block_info[2].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 3u};
fixture.program.block_info[3].terminator = {
.kind = variant == Variant::PreviousBound ? CFG::TerminatorKind::ConditionalBranch
: CFG::TerminatorKind::Branch,
.true_block = 1u, .false_block = 5u};
fixture.program.block_info[4].terminator = {
.kind = final_exit != exit || variant == Variant::ExitBypass ||
variant == Variant::IncrementBypass
? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
.true_block = final_exit != exit || variant == Variant::IncrementBypass ? 3u : 2u};
if (final_exit != exit)
fixture.program.block_info[5].terminator.kind = CFG::TerminatorKind::Return;
}
auto &phi = header->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U32));
const auto key = Value(&phi);
const auto count = fixture.UserData(2);
auto *previous = variant == Variant::PreviousBound ?
&header->AppendNewInst(ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1)) : nullptr;
const auto local = fixture.Emit(
ValueOpcode::GetBuiltin,
{Value(static_cast<uint32_t>(diamond ? StageInputKind::VertexIndex
: StageInputKind::LocalInvocationId)), Value(0u)});
const auto count = variant == Variant::DivergentBound ? local : fixture.UserData(2);
const auto in_range = fixture.Emit(ValueOpcode::SLessThan32,
{variant == Variant::WrongGuard
? Value(0u) : key,
count}, 0, header);
const auto allowed = fixture.Emit(ValueOpcode::LogicalAnd,
{in_range, Value(true)}, 0, header);
fixture.program.block_info[1].condition =
fixture.Emit(ValueOpcode::LogicalNot, {allowed}, 0, header);
count}, 0, compare);
const auto initial_active = fixture.Emit(ValueOpcode::INotEqual32,
{local, Value(0u)}, 0, entry);
if (masked) {
auto *active_phi = diamond ? nullptr : &header->AppendNewInst(
ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
auto &saved_phi = header->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U1));
auto &status_phi = header->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U32));
const auto saved = Value(&saved_phi);
const auto status = Value(&status_phi);
const auto active = diamond ? fixture.Emit(ValueOpcode::LogicalAnd,
{saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32,
{Value(0u), status}, 0, header)}, 0, header) : Value(active_phi);
const auto mask = fixture.Emit(ValueOpcode::LogicalOr,
{fixture.Emit(ValueOpcode::LogicalAnd, {in_range, active}, 0, compare),
fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, compare)}, 0, compare);
auto next_saved = fixture.Emit(ValueOpcode::LogicalAnd,
{variant == Variant::MaskResurrection ? Value(true) : saved, mask}, 0, compare);
const auto execute = fixture.Emit(ValueOpcode::LogicalAnd, {active, mask}, 0, compare);
const auto body_status = fixture.Emit(ValueOpcode::SelectU32,
{execute, local, status}, 0, guard);
const auto next_active = fixture.Emit(ValueOpcode::LogicalAnd,
{next_saved, fixture.Emit(ValueOpcode::UGreaterThanEqual32,
{Value(3u), body_status}, 0, latch)}, 0, latch);
auto next_status = fixture.Emit(ValueOpcode::SelectU32,
{variant == Variant::StatusOverwrite ? Value(true) : next_active,
Value(0u), body_status}, 0,
variant == Variant::GuardedDiamondBypass ? latch : increment);
if (variant == Variant::GuardedDiamondBypass) {
auto &merged_saved = increment->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U1));
merged_saved.AddPhiOperand(compare, saved);
merged_saved.AddPhiOperand(latch, next_saved);
next_saved = Value(&merged_saved);
auto &merged_status = increment->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U32));
merged_status.AddPhiOperand(compare, status);
merged_status.AddPhiOperand(latch, next_status);
next_status = Value(&merged_status);
}
if (active_phi != nullptr) {
active_phi->AddPhiOperand(entry, initial_active);
active_phi->AddPhiOperand(increment, next_active);
}
saved_phi.AddPhiOperand(entry, initial_active);
saved_phi.AddPhiOperand(increment, next_saved);
status_phi.AddPhiOperand(entry, diamond ? Value(0u) : local);
status_phi.AddPhiOperand(increment, next_status);
const auto branch = [&](uint32_t index, uint32_t yes, uint32_t no,
Value predicate, CFG::BranchCondition kind) {
auto *block = fixture.program.blocks[index];
block->AddBranch(fixture.program.blocks[yes]);
block->AddBranch(fixture.program.blocks[no]);
const auto inverse = fixture.Emit(ValueOpcode::LogicalNot, {predicate}, 0, block);
fixture.program.block_info[index].condition =
fixture.Emit(ValueOpcode::ConditionRef, {inverse}, kind, block);
fixture.program.block_info[index].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = yes, .false_block = no};
};
fixture.program.block_info[0].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 1u};
branch(1u, 4u, 5u, active, CFG::BranchCondition::ExecZero);
branch(5u, variant == Variant::GuardedDiamondBypass ? 7u : 4u, 6u,
diamond ? execute : mask,
diamond ? CFG::BranchCondition::ExecZero : CFG::BranchCondition::VccZero);
const auto image_guard = variant == Variant::MaskedWrongGuard ? active :
fixture.Emit(ValueOpcode::LogicalAnd,
{fixture.Emit(ValueOpcode::LogicalAnd, {execute, in_range}, 0, guard),
fixture.Emit(ValueOpcode::IEqual32, {local, Value(1u)}, 0, guard)}, 0, guard);
branch(6u, 3u, 2u, image_guard, CFG::BranchCondition::ExecZero);
body->AddBranch(latch);
fixture.program.block_info[2].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 3u};
branch(3u, 4u, 7u, next_active, CFG::BranchCondition::ExecZero);
increment->AddBranch(header);
fixture.program.block_info[7].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 1u};
fixture.program.block_info[4].terminator.kind = CFG::TerminatorKind::Return;
} else {
const auto active = initial_active;
const auto allowed = fixture.Emit(variant == Variant::Disjunction
? ValueOpcode::LogicalOr : ValueOpcode::LogicalAnd,
{in_range, active}, 0, header);
const bool nonzero = variant == Variant::Nonzero || variant == Variant::TrueEdge;
auto condition = allowed;
if (!nonzero)
condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header);
if (variant != Variant::Plain)
condition = fixture.Emit(ValueOpcode::ConditionRef, {condition},
nonzero ? CFG::BranchCondition::ExecNonZero
: CFG::BranchCondition::ExecZero, header);
if (variant == Variant::Nonzero || variant == Variant::WrongPolarity)
condition = fixture.Emit(ValueOpcode::LogicalNot, {condition}, 0, header);
fixture.program.block_info[1].condition = condition;
if (variant == Variant::PreviousBound) {
previous->AddPhiOperand(entry, Value(false));
previous->AddPhiOperand(latch, in_range);
const auto continuing = fixture.Emit(ValueOpcode::LogicalOr,
{in_range, Value(previous)}, 0, latch);
fixture.program.block_info[3].condition = fixture.Emit(ValueOpcode::ConditionRef,
{continuing}, CFG::BranchCondition::SccNonZero, latch);
}
}
const auto step = fixture.Emit(ValueOpcode::IAdd32,
{key, Value(variant == Variant::WrongStep ? 2u : 1u)},
0, latch);
phi.AddPhiOperand(entry, Value(0u));
phi.AddPhiOperand(latch, step);
0, increment);
phi.AddPhiOperand(entry, variant == Variant::DivergentKey ? local : Value(0u));
phi.AddPhiOperand(increment, step);
fixture.block = variant == Variant::GuardBlock ? header : body;
const auto table = fixture.Address(fixture.UserData(0), fixture.UserData(1));
@@ -787,6 +975,18 @@ void TestBoundedComputeImageLoop() {
};
auto plan = make_plan(Variant::Bounded);
make_plan(Variant::Plain);
make_plan(Variant::Nonzero);
make_plan(Variant::TrueEdge);
make_plan(Variant::Masked);
make_plan(Variant::GuardedDiamond);
for (const auto variant : {Variant::IncrementBypass, Variant::PreviousBound,
Variant::MaskedWrongGuard,
Variant::MaskResurrection, Variant::StatusOverwrite,
Variant::GuardedDiamondBypass}) {
CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
"compute image loop allowed an unbounded induction or mask resurrection");
}
CheckFatal([&] { make_plan(Variant::WrongGuard); },
"not a valid runtime value",
"compute image loop accepted an unrelated guard");
@@ -802,6 +1002,11 @@ void TestBoundedComputeImageLoop() {
CheckFatal([&] { make_plan(Variant::WrongStep); },
"not a valid runtime value",
"compute image loop accepted a two-step induction");
for (const auto variant : {Variant::DivergentBound, Variant::DivergentKey,
Variant::Disjunction, Variant::WrongPolarity}) {
CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
"compute image loop accepted a guard without a uniform bound");
}
LinearTestMemory memory;
const auto table = 0x1800u + 0x6b0u;
@@ -849,8 +1054,21 @@ void TestBoundedComputeImageLoop() {
void TestUniformizedMaterialImageKeys() {
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
const auto make_plan = [](bool wrong_update, bool wrong_equality) {
enum class Variant {
Valid, Plain, WrongUpdate, WrongEquality, WrongExit, WrongCarry, WrongBackedge,
AndNot, Subset, FirstLane, FirstLaneDirect, FirstLaneAndNot, EmptyFirstEntry, WrongFirstBackedge,
WideningFirstEntry, WideningFirstBackedge, WrongFirstLoadMask
};
const auto make_plan = [](Variant variant) {
Fixture fixture;
fixture.program.wave_size = 64u;
const bool first_lane = variant >= Variant::FirstLane;
const bool first_lane_loop = first_lane && variant != Variant::FirstLaneDirect;
const uint32_t sample_entry_id = first_lane_loop ? 10u : 9u;
const auto branch = [&](Value predicate, CFG::BranchCondition kind, Block *block) {
return variant == Variant::Plain ? predicate : fixture.Emit(
ValueOpcode::ConditionRef, {predicate}, kind, block);
};
auto *entry = fixture.block;
auto *header = fixture.AddBlock();
auto *inactive = fixture.AddBlock();
@@ -860,6 +1078,8 @@ void TestUniformizedMaterialImageKeys() {
auto *choose = fixture.AddBlock();
auto *sample = fixture.AddBlock();
auto *done = fixture.AddBlock();
auto *sample_header = first_lane_loop ? fixture.AddBlock() : nullptr;
auto *sample_entry = first_lane ? fixture.AddBlock() : nullptr;
entry->AddBranch(header);
header->AddBranch(inactive);
inactive->AddBranch(merge);
@@ -869,23 +1089,44 @@ void TestUniformizedMaterialImageKeys() {
bit->AddBranch(header);
bit->AddBranch(merge);
merge->AddBranch(choose);
choose->AddBranch(sample);
choose->AddBranch(first_lane ? sample_entry : sample);
choose->AddBranch(done);
if (first_lane) {
sample_entry->AddBranch(first_lane_loop ? sample_header : sample);
fixture.program.block_info[sample_entry_id].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = first_lane_loop ? 9u : 7u};
}
if (first_lane_loop) {
sample_header->AddBranch(sample);
sample->AddBranch(sample_header);
fixture.program.block_info[9].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 7u,
.merge_block = 8u, .continue_block = 7u, .loop_header = true};
}
sample->AddBranch(done);
fixture.program.block_info[0].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 1u};
fixture.program.block_info[1].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 2u};
const auto active_on_entry = fixture.Emit(
const auto enabled = fixture.Emit(
ValueOpcode::INotEqual32, {fixture.UserData(5u), Value(0u)}, 0, entry);
const auto local = fixture.Emit(ValueOpcode::GetBuiltin,
{Value(static_cast<uint32_t>(StageInputKind::LocalInvocationId)), Value(0u)}, 0, entry);
const auto active_on_entry = fixture.Emit(ValueOpcode::LogicalAnd,
{enabled, fixture.Emit(ValueOpcode::INotEqual32, {local, Value(0u)}, 0, entry)},
0, entry);
auto &active_phi = header->AppendNewInst(
ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
auto &mask_phi = header->AppendNewInst(
ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
auto &carry_phi = header->AppendNewInst(
ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
const auto mask = Value(&mask_phi);
const auto active = Value(&active_phi);
fixture.program.block_info[2].condition = fixture.Emit(
ValueOpcode::LogicalNot, {active}, 0, inactive);
fixture.program.block_info[2].condition = branch(
fixture.Emit(ValueOpcode::LogicalNot, {active}, 0, inactive),
variant == Variant::WrongExit ? CFG::BranchCondition::ExecNonZero
: CFG::BranchCondition::ExecZero, inactive);
fixture.program.block_info[2].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 5u, .false_block = 3u};
@@ -893,8 +1134,9 @@ void TestUniformizedMaterialImageKeys() {
ValueOpcode::INotEqual32, {Value(0u), mask}, 0, sentinel);
const auto bit_guard = fixture.Emit(
ValueOpcode::LogicalAnd, {active, nonzero}, 0, sentinel);
fixture.program.block_info[3].condition = fixture.Emit(
ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel);
fixture.program.block_info[3].condition = branch(
fixture.Emit(ValueOpcode::LogicalNot, {bit_guard}, 0, sentinel),
CFG::BranchCondition::ExecZero, sentinel);
fixture.program.block_info[3].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 5u, .false_block = 4u};
@@ -903,12 +1145,19 @@ void TestUniformizedMaterialImageKeys() {
ValueOpcode::BitwiseAnd32, {first, Value(31u)}, 0, bit);
const auto one_bit = fixture.Emit(
ValueOpcode::ShiftLeftLogical32, {Value(1u), position}, 0, bit);
const bool and_not = variant == Variant::AndNot || variant == Variant::FirstLaneAndNot;
const auto removed = and_not
? fixture.Emit(ValueOpcode::BitwiseNot32, {one_bit}, 0, bit)
: variant == Variant::Subset ? fixture.UserData(7u) : one_bit;
const auto cleared = fixture.Emit(
wrong_update ? ValueOpcode::BitwiseOr32 : ValueOpcode::BitwiseXor32,
{mask, one_bit}, 0, bit);
variant == Variant::WrongUpdate ? ValueOpcode::BitwiseOr32
: and_not || variant == Variant::Subset ? ValueOpcode::BitwiseAnd32
: ValueOpcode::BitwiseXor32,
{mask, removed}, 0, bit);
const auto continuation = fixture.Emit(
ValueOpcode::LogicalAnd, {bit_guard, active_on_entry}, 0, bit);
fixture.program.block_info[4].condition = continuation;
fixture.program.block_info[4].condition = branch(
continuation, CFG::BranchCondition::ExecNonZero, bit);
fixture.program.block_info[4].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 1u, .false_block = 5u};
@@ -917,13 +1166,18 @@ void TestUniformizedMaterialImageKeys() {
mask_phi.AddPhiOperand(entry, fixture.UserData(4u));
mask_phi.AddPhiOperand(bit, cleared);
const auto arbitrary = fixture.UserData(6u);
const auto carry = Value(&carry_phi);
const auto sentinel_index = fixture.Emit(
ValueOpcode::SelectU32, {active, Value(32u), arbitrary}, 0, sentinel);
ValueOpcode::SelectU32,
{active, Value(32u), variant == Variant::WrongCarry ? arbitrary : carry},
0, sentinel);
const auto bit_index = fixture.Emit(
ValueOpcode::SelectU32, {bit_guard, first, sentinel_index}, 0, bit);
carry_phi.AddPhiOperand(entry, arbitrary);
carry_phi.AddPhiOperand(bit, variant == Variant::WrongBackedge ? arbitrary : bit_index);
auto &index_phi = merge->AppendNewInst(
ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U32));
index_phi.AddPhiOperand(inactive, arbitrary);
index_phi.AddPhiOperand(inactive, carry);
index_phi.AddPhiOperand(sentinel, sentinel_index);
index_phi.AddPhiOperand(bit, bit_index);
const auto index = Value(&index_phi);
@@ -950,24 +1204,54 @@ void TestUniformizedMaterialImageKeys() {
const auto base = fixture.Address(fixture.UserData(0u), fixture.UserData(1u));
MemoryInfo material_memory;
material_memory.kind = ResourceKind::Global;
const auto load_mask = variant == Variant::WrongFirstLoadMask
? fixture.Emit(ValueOpcode::LogicalNot, {material_guard}, 0, choose)
: material_guard;
const auto loaded = fixture.Emit(
ValueOpcode::LoadAddressU32,
{base, material_offset, Value(0u), material_guard},
{base, material_offset, Value(0u), load_mask},
fixture.AddMemory(material_memory, 0x1a88u), choose);
const auto local = fixture.Emit(
const auto local_key = fixture.Emit(
ValueOpcode::SelectU32, {material_guard, loaded, arbitrary}, 0, choose);
const auto key = fixture.Emit(
ValueOpcode::ReadLane, {local, Value(0u)}, 0, choose);
Inst *sample_active_phi = nullptr;
auto sample_active = material_guard;
if (first_lane) {
const auto initial = variant == Variant::EmptyFirstEntry ? Value(false)
: variant == Variant::WideningFirstEntry ? enabled
: fixture.Emit(ValueOpcode::LogicalAnd,
{material_guard, fixture.Emit(ValueOpcode::SLessThanEqual32,
{Value(0u), local_key}, 0, choose)}, 0, choose);
sample_active = initial;
if (first_lane_loop) {
sample_active_phi = &sample_header->AppendNewInst(
ValueOpcode::Phi, {}, static_cast<uint64_t>(Type::U1));
sample_active_phi->AddPhiOperand(sample_entry, initial);
sample_active = Value(sample_active_phi);
}
fixture.program.block_info[6].condition = branch(
variant == Variant::EmptyFirstEntry ? enabled : initial,
CFG::BranchCondition::ExecNonZero, choose);
fixture.program.block_info[6].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = sample_entry_id, .false_block = 8u};
}
auto *key_block = first_lane ? sample : choose;
const auto key = first_lane
? fixture.Emit(ValueOpcode::ReadFirstLane, {local_key, sample_active}, 0, key_block)
: fixture.Emit(ValueOpcode::ReadLane, {local_key, Value(0u)}, 0, key_block);
const auto compared = fixture.Emit(
ValueOpcode::IEqual32,
{key, wrong_equality ? arbitrary : local}, 0, choose);
{key, variant == Variant::WrongEquality ? arbitrary : local_key}, 0, key_block);
const auto sample_guard = fixture.Emit(
ValueOpcode::LogicalAnd, {material_guard, compared}, 0, choose);
fixture.program.block_info[6].condition = fixture.Emit(
ValueOpcode::LogicalNot, {sample_guard}, 0, choose);
fixture.program.block_info[6].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 8u, .false_block = 7u};
ValueOpcode::LogicalAnd, {sample_active, compared}, 0, key_block);
if (!first_lane) {
fixture.program.block_info[6].condition = branch(
fixture.Emit(ValueOpcode::LogicalNot, {sample_guard}, 0, choose),
CFG::BranchCondition::ExecZero, choose);
fixture.program.block_info[6].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 8u, .false_block = 7u};
}
const auto table_offset = fixture.Emit(
ValueOpcode::IAdd32,
{fixture.Emit(ValueOpcode::ShiftLeftLogical32,
@@ -997,10 +1281,33 @@ void TestUniformizedMaterialImageKeys() {
MemoryInfo image_memory;
image_memory.kind = ResourceKind::Image;
image_memory.image_dimension = Decoder::ImageDimension::Dim2D;
fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
fixture.AddMemory(image_memory, 0x1aecu), sample);
fixture.program.block_info[7].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 8u};
const auto sampled = fixture.Emit(ValueOpcode::ImageSampleRaw, {image, sampler, address},
fixture.AddMemory(image_memory, 0x1aecu), sample);
if (first_lane) {
const auto component = fixture.Emit(
ValueOpcode::CompositeExtractU32x4, {sampled, Value(0u)}, 0, sample);
fixture.Emit(ValueOpcode::ReferenceU32,
{fixture.Emit(ValueOpcode::SelectU32,
{variant == Variant::EmptyFirstEntry ? material_guard : sample_guard,
component, Value(0u)}, 0, sample)}, 0, sample);
}
if (first_lane_loop) {
auto remaining = fixture.Emit(ValueOpcode::LogicalAnd,
{sample_active, fixture.Emit(ValueOpcode::LogicalNot,
{sample_guard}, 0, sample)}, 0, sample);
if (variant == Variant::WideningFirstBackedge)
remaining = fixture.Emit(ValueOpcode::LogicalOr, {remaining, enabled}, 0, sample);
sample_active_phi->AddPhiOperand(sample, remaining);
fixture.program.block_info[7].condition = branch(
variant == Variant::WrongFirstBackedge ? sample_guard : remaining,
CFG::BranchCondition::ExecNonZero, sample);
fixture.program.block_info[7].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 9u, .false_block = 8u};
} else {
fixture.program.block_info[7].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 8u};
}
fixture.program.block_info[8].terminator.kind = CFG::TerminatorKind::Return;
const auto output = fixture.Emit(
ValueOpcode::GetBufferResource,
@@ -1024,7 +1331,13 @@ void TestUniformizedMaterialImageKeys() {
"uniformized image key lost its finite material range");
return ExtractResourcePlan(fixture.program);
};
auto plan = make_plan(false, false);
auto plan = make_plan(Variant::Valid);
make_plan(Variant::Plain);
make_plan(Variant::AndNot);
make_plan(Variant::Subset);
make_plan(Variant::FirstLane);
make_plan(Variant::FirstLaneDirect);
plan = make_plan(Variant::FirstLaneAndNot);
Check(plan.requires_specialization_memory &&
plan.descriptor_sources[plan.info.images[0].source]
.indirect_image->selector_mask.Resolve().TryInstruction() != nullptr,
@@ -1078,10 +1391,22 @@ void TestUniformizedMaterialImageKeys() {
user_data[8] = first_table;
Check(!MaterializeResources(plan, runtime, snapshot, specialization),
"written buffer alias with an image record was accepted");
CheckFatal([&] { make_plan(true, false); }, "not a valid runtime value",
CheckFatal([&] { make_plan(Variant::WrongUpdate); }, "not a valid runtime value",
"non-clearing material mask was accepted");
CheckFatal([&] { make_plan(false, true); }, "not a valid runtime value",
CheckFatal([&] { make_plan(Variant::WrongEquality); }, "not a valid runtime value",
"unrelated ReadLane key was accepted");
CheckFatal([&] { make_plan(Variant::WrongExit); }, "not a valid runtime value",
"one inactive lane bypassed material index initialization for active lanes");
for (const auto variant : {Variant::WrongCarry, Variant::WrongBackedge}) {
CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
"inactive material lane did not preserve its selected index across iterations");
}
for (const auto variant : {Variant::EmptyFirstEntry, Variant::WrongFirstBackedge,
Variant::WideningFirstEntry, Variant::WideningFirstBackedge,
Variant::WrongFirstLoadMask}) {
CheckFatal([&] { make_plan(variant); }, "not a valid runtime value",
"first-lane material key escaped its active load mask");
}
}
void TestImageDescriptorFields() {
@@ -1405,7 +1730,7 @@ void TestDenseBufferTracking() {
fixture.program.memory_info[other_flags.index].resource == 1,
"typed memory metadata was not patched to dense indices");
CheckFatal([&] { TrackResources(fixture.program); }, "already tracked",
CheckFatal([&] { fixture.PlanAndTrack(); }, "already tracked",
"resource tracking allowed a second mutation pass");
}
@@ -1601,8 +1926,8 @@ void TestSampleAdjustSamplerScratch() {
rejected.Emit(ValueOpcode::ImageSampleRaw,
{rejected_image, rejected_sampler, rejected.ImageAddress()},
rejected.AddMemory(rejected_memory, 0x200));
BuildSrtPlan(rejected.program);
CheckFatal([&] { TrackResources(rejected.program); },
CheckFatal([&] { rejected.PlanAndTrack(); },
"not a valid runtime value", message);
};
CheckRejected(0u, 12u,
@@ -1936,7 +2261,8 @@ void TestDynamicSrtReadRemainsExplicit() {
fixture.PlanAndTrack();
Check(fixture.program.srt_reads.empty() &&
fixture.program.dynamic_reads.size() == 1 &&
read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 &&
!fixture.program.memory_info[0].planning_only &&
fixture.program.info.uses_dma,
"dynamic scalar read was incorrectly flattened or lost");
std::array<uint32_t, 3> user_data{0x1000u, 0u, 4u};
@@ -1990,8 +2316,7 @@ void TestPhiValidation() {
{handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(memory, 20), merge);
BuildSrtPlan(fixture.program);
CheckFatal([&] { TrackResources(fixture.program); }, "not a valid runtime value",
CheckFatal([&] { fixture.PlanAndTrack(); }, "not a valid runtime value",
"control-dependent descriptor phi was accepted");
Check(!fixture.program.resource_tracking_complete &&
fixture.program.info.buffers.empty() &&
@@ -2092,7 +2417,9 @@ ResourcePlan ConditionalSamplerPlan(bool diamond, bool reverse, bool reverse_phi
{result, Value(0u)})});
}
fixture.PlanAndTrack();
Check(fixture.program.value_storage.size() == 4,
Check(std::ranges::count_if(fixture.program.value_storage, [](const Inst &inst) {
return inst.GetOpcode() == ValueOpcode::SelectU32;
}) == 4,
"repeated sampler uses retained duplicate planning selections");
Check(sampler_words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
"host descriptor selection changed the GPU Phi");
@@ -2155,90 +2482,6 @@ void TestConditionalSamplerPhi() {
}
}
void TestGuardedSamplerPhi() {
namespace CFG = Libs::Graphics::ShaderRecompiler::CFG;
const auto make_plan = [](bool take_true, bool reverse_phi, bool bypass,
bool mismatched, bool ambiguous) {
Fixture fixture(ShaderType::Pixel);
auto *entry = fixture.block;
auto *loaded = fixture.AddBlock();
auto *merge = fixture.AddBlock();
auto *sample = fixture.AddBlock();
auto *exit = fixture.AddBlock();
entry->AddBranch(loaded);
entry->AddBranch(merge);
loaded->AddBranch(merge);
merge->AddBranch(sample);
merge->AddBranch(exit);
sample->AddBranch(exit);
if (bypass) exit->AddBranch(sample);
const auto lane = fixture.Emit(ValueOpcode::LaneId);
const auto predicate = fixture.Emit(ValueOpcode::IEqual32, {lane, Value(0u)});
fixture.program.block_info[0].condition = predicate;
fixture.program.block_info[0].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = 1u, .false_block = 2u};
fixture.program.block_info[1].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 2u};
auto &guard = merge->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U1));
guard.AddPhiOperand(entry, ambiguous ? predicate : Value(!take_true));
guard.AddPhiOperand(mismatched ? sample : loaded, predicate);
fixture.program.block_info[2].condition = Value(&guard);
fixture.program.block_info[2].terminator = {
.kind = CFG::TerminatorKind::ConditionalBranch,
.true_block = take_true ? 3u : 4u,
.false_block = take_true ? 4u : 3u};
fixture.program.block_info[3].terminator = {
.kind = CFG::TerminatorKind::Branch, .true_block = 4u};
fixture.program.block_info[4].terminator = {
.kind = bypass ? CFG::TerminatorKind::Branch : CFG::TerminatorKind::Return,
.true_block = 3u};
std::array<Value, 4> words;
for (uint32_t word = 0; word < words.size(); ++word) {
// An arbitrary excluded value proves this is control-flow reasoning, not null filtering.
const auto first = Value(0xbad000u + word);
const auto second = fixture.UserData(word);
auto &phi = merge->AppendNewInst(ValueOpcode::Phi, {},
static_cast<uint64_t>(Type::U32));
phi.AddPhiOperand(reverse_phi ? loaded : entry, reverse_phi ? second : first);
phi.AddPhiOperand(reverse_phi ? entry : loaded, reverse_phi ? first : second);
words[word] = Value(&phi);
}
fixture.block = sample;
const auto image = fixture.Image({Value(0u), Value(0u), Value(0u), Value(0u),
Value(0u), Value(0u), Value(0u), Value(0u)});
const auto sampler = fixture.Sampler(words);
MemoryInfo memory;
memory.kind = ResourceKind::Image;
memory.image_dimension = Decoder::ImageDimension::Dim2D;
fixture.Emit(ValueOpcode::ImageSampleRaw,
{image, sampler, fixture.ImageAddress()},
fixture.AddMemory(memory, 0x2a0));
fixture.PlanAndTrack();
Check(words[0].ResolveInstruction()->GetOpcode() == ValueOpcode::Phi,
"guarded host descriptor selection rewrote the GPU Phi");
return ExtractResourcePlan(fixture.program);
};
for (const bool take_true : {false, true}) {
for (const bool reverse_phi : {false, true}) {
auto plan = make_plan(take_true, reverse_phi, false, false, false);
const std::array<uint32_t, 4> user_data{0x444u, 0x555u, 0x666u, 0x777u};
DescriptorValue selected;
Check(SrtWalker(plan, {.user_data = user_data}).EvaluateDescriptor(
plan.info.samplers[0].source, selected) &&
std::equal(user_data.begin(), user_data.end(), selected.dwords.begin()),
"guarded sampler did not match descriptor and predicate predecessors");
}
}
CheckFatal([&] { make_plan(true, false, true, false, false); },
"not a valid runtime value", "sampler guard accepted an unguarded path");
CheckFatal([&] { make_plan(true, false, false, true, false); },
"not a valid runtime value", "sampler guard ignored predecessor identity");
CheckFatal([&] { make_plan(true, false, false, false, true); },
"not a valid runtime value", "sampler guard discarded a reachable alternative");
}
void TestLoopCycleEnteredThroughRuntimeValue() {
Fixture fixture;
auto *entry = fixture.block;
@@ -2252,11 +2495,24 @@ void TestLoopCycleEnteredThroughRuntimeValue() {
{Value(&phi), Value(0xffffffffu)}, 0, loop);
phi.AddPhiOperand(entry, initial);
phi.AddPhiOperand(loop, carried);
fixture.Emit(ValueOpcode::GetBufferResource,
const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
{carried, Value(0u), Value(0u), Value(0u)}, MemoryFlags{0, 12},
loop);
BuildSrtPlan(fixture.program);
MemoryInfo memory;
memory.kind = ResourceKind::Buffer;
fixture.Emit(ValueOpcode::LoadBufferU32,
{handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(memory, 16), loop);
ConstantPropagationPass(fixture.program.blocks);
RemoveIdentities(fixture.program.blocks);
fixture.PlanAndTrack();
const std::array<uint32_t, 1> user_data{0x4000u};
DescriptorValue descriptor;
Check(fixture.program.info.buffers.size() == 1u &&
SrtWalker(fixture.program, {.user_data = user_data}).EvaluateDescriptor(
fixture.program.info.buffers[0].source, descriptor) &&
descriptor.dwords[0] == user_data[0],
"runtime-rooted invariant loop lost its buffer source");
}
void TestInvariantLoopPhi() {
@@ -2600,14 +2856,15 @@ void TestShaderInfoAndBindingLayout() {
fixture.Emit(ValueOpcode::LoadBufferU32,
{handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(buffer, 4));
fixture.Emit(
const auto invocation = fixture.Emit(
ValueOpcode::GetBuiltin,
{Value(static_cast<uint32_t>(StageInputKind::GlobalInvocationId)),
Value(2u)});
fixture.Emit(ValueOpcode::BitwiseXor32, {Value(1u), Value(2u)});
const auto value = fixture.Emit(ValueOpcode::BitwiseXor32,
{invocation, Value(2u)});
MemoryInfo gds;
gds.kind = ResourceKind::Gds;
fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), Value(1u), Value(true)},
fixture.Emit(ValueOpcode::WriteSharedU32, {Value(0u), value, Value(true)},
fixture.AddMemory(gds, 8));
fixture.PlanAndTrack();
@@ -2851,8 +3108,8 @@ void TestResourceLimitIsTransactional() {
{handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(memory, index * 4u));
}
BuildSrtPlan(fixture.program);
CheckFatal([&] { TrackResources(fixture.program); },
CheckFatal([&] { fixture.PlanAndTrack(); },
"buffer resource limit exceeded",
"resource-limit failure was not reported");
Check(!fixture.program.resource_tracking_complete &&
@@ -2870,9 +3127,9 @@ void TestMalformedMemoryKindsRejected() {
fixture.Emit(ValueOpcode::StoreAddressU32,
{address, Value(0u), Value(0u), Value(1u), Value(true)},
fixture.AddMemory(memory, 4));
BuildSrtPlan(fixture.program);
CheckFatal(
[&] { TrackResources(fixture.program); },
[&] { fixture.PlanAndTrack(); },
"address operation has invalid resource kind",
"resource tracking accepted an address opcode with buffer metadata");
}
@@ -2887,9 +3144,9 @@ void TestMalformedMemoryKindsRejected() {
fixture.Emit(ValueOpcode::ImageRead,
{image, fixture.ImageAddress(), Value(true)},
fixture.AddMemory(memory, 8));
BuildSrtPlan(fixture.program);
CheckFatal(
[&] { TrackResources(fixture.program); },
[&] { fixture.PlanAndTrack(); },
"image operation has invalid resource kind",
"resource tracking accepted an image opcode with address metadata");
}
@@ -2924,7 +3181,6 @@ int main() {
Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit);
Run("phi validation", TestPhiValidation);
Run("conditional sampler phi", TestConditionalSamplerPhi);
Run("guarded sampler phi", TestGuardedSamplerPhi);
Run("runtime-rooted loop", TestLoopCycleEnteredThroughRuntimeValue);
Run("invariant loop phi", TestInvariantLoopPhi);
Run("DMA address materialization", TestDmaAddressMaterialization);
@@ -2981,3 +3237,4 @@ void DbgExit(int) { throw std::runtime_error("typed IR assertion failed"); }
#include "graphics/shader/recompiler/ir/Value.cpp"
#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
+98 -50
View File
@@ -2,6 +2,7 @@
#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include <cstdlib>
@@ -71,7 +72,7 @@ struct Fixture {
return static_cast<uint32_t>(program.memory_info.size() - 1u);
}
void Plan() { BuildSrtPlan(program); }
void Plan() { TrackResources(program, {}, {}); }
};
struct TestMemory {
@@ -102,6 +103,16 @@ Value RawRead(Fixture &fixture, Value address, Value offset, uint32_t memory,
0x80, block);
}
void LoadBuffer(Fixture &fixture, std::array<Value, 4> words) {
const auto handle = fixture.Emit(ValueOpcode::GetBufferResource,
{words[0], words[1], words[2], words[3]});
const auto loaded = fixture.EmitMemory(
ValueOpcode::LoadBufferU32,
{handle, Value(0u), Value(0u), Value(0u), Value(true)},
fixture.AddMemory(ResourceKind::Buffer));
fixture.Emit(ValueOpcode::ReferenceU32, {loaded});
}
void TestImmediateFlatteningAndGvn() {
Fixture fixture;
const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 0x20);
@@ -109,14 +120,11 @@ void TestImmediateFlatteningAndGvn() {
fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
const auto second = RawRead(
fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
fixture.Emit(ValueOpcode::GetBufferResource,
{first, second, Value(16u), Value(0u)});
LoadBuffer(fixture, {first, second, Value(16u), Value(0u)});
fixture.Plan();
Check(fixture.program.srt_reads.size() == 1,
"equivalent typed scalar reads were not coalesced");
Check(fixture.program.dynamic_reads.empty(),
"immediate scalar read was classified as dynamic");
Check(fixture.program.memory_info[memory].planning_only,
"flattened raw read was not kept as a planning-only root");
@@ -133,8 +141,7 @@ void TestRawScalarComponentAlignment() {
const auto memory = fixture.AddMemory(ResourceKind::ScalarAddress, 2);
const auto read = RawRead(
fixture, Address(fixture, Value(0x1003u), Value(0u)), Value(2u), memory);
fixture.Emit(ValueOpcode::GetBufferResource,
{read, Value(0u), Value(16u), Value(0u)});
LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
fixture.Plan();
TestMemory memory_image{{{0x1000u, 0x12345678u}}};
@@ -150,7 +157,7 @@ void TestScalarMemoryDomainMismatchFails() {
Fixture raw;
const auto raw_memory = raw.AddMemory(ResourceKind::ScalarBuffer);
RawRead(raw, Address(raw, Value(0x1000u), Value(0u)), Value(0u), raw_memory);
CheckFatal([&] { BuildSrtPlan(raw.program); },
CheckFatal([&] { TrackResources(raw.program, {}, {}); },
"incompatible scalar memory metadata",
"raw scalar load accepted descriptor-buffer metadata");
@@ -161,7 +168,7 @@ void TestScalarMemoryDomainMismatchFails() {
{Value(0x1000u), Value(0u), Value(16u), Value(0u)});
buffer.EmitMemory(ValueOpcode::ReadConstBuffer, {resource, Value(0u)},
buffer_memory);
CheckFatal([&] { BuildSrtPlan(buffer.program); },
CheckFatal([&] { TrackResources(buffer.program, {}, {}); },
"incompatible scalar memory metadata",
"descriptor scalar load accepted raw-address metadata");
}
@@ -173,12 +180,12 @@ void TestDynamicReadRemainsTyped() {
{Value(static_cast<ScalarReg>(2))});
const auto read = RawRead(
fixture, Address(fixture, Value(0x1000u), Value(0u)), offset, memory);
fixture.Emit(ValueOpcode::GetBufferResource,
{read, Value(0u), Value(16u), Value(0u)});
LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
fixture.Plan();
Check(fixture.program.srt_reads.empty() &&
fixture.program.dynamic_reads == std::vector<Value>{read},
read.ResolveInstruction()->GetOpcode() == ValueOpcode::LoadAddressU32 &&
!fixture.program.memory_info[memory].planning_only,
"dynamic scalar read received a fake flattened slot");
}
@@ -189,8 +196,7 @@ void TestNestedSrtWalk() {
fixture, Address(fixture, Value(0x1000u), Value(0u)), Value(0u), memory);
const auto value =
RawRead(fixture, Address(fixture, pointer, Value(0u)), Value(0u), memory);
fixture.Emit(ValueOpcode::GetBufferResource,
{value, Value(0u), Value(16u), Value(0u)});
LoadBuffer(fixture, {value, Value(0u), Value(16u), Value(0u)});
fixture.Plan();
TestMemory memory_image{{{0x1000u, 0x2000u}, {0x2000u, 0xabcdef01u}}};
@@ -211,7 +217,7 @@ void TestShaderBaseAndUserData() {
const auto user = fixture.Emit(ValueOpcode::GetUserData,
{Value(static_cast<ScalarReg>(2))});
const auto sum = fixture.Emit(ValueOpcode::IAdd32, {user, Value(4u)});
fixture.Plan();
fixture.program.descriptor_sources.push_back(
{.dwords = {low, high, sum}, .dword_count = 3});
@@ -239,7 +245,7 @@ void TestCarryAndBitFields() {
{Value(0u), Value(0x89abcdefu), Value(0u), Value(32u)});
const auto sign = fixture.Emit(ValueOpcode::BitFieldSExtract,
{Value(0x000000f0u), Value(4u), Value(4u)});
fixture.Plan();
fixture.program.descriptor_sources.push_back(
{.dwords = {low, high, inserted, sign}, .dword_count = 4});
@@ -261,7 +267,7 @@ void TestInvariantAndDivergentPhi() {
divergent.SetFlags(Type::U32);
divergent.AddPhiOperand(&fixture.BlockAt(0), Value(7u));
divergent.AddPhiOperand(&fixture.BlockAt(1), Value(9u));
fixture.Plan();
fixture.program.descriptor_sources.push_back(
{.dwords = {Value(&invariant)}, .dword_count = 1});
fixture.program.descriptor_sources.push_back(
@@ -285,6 +291,7 @@ void TestControlDependentStandaloneLoadStaysTyped() {
const auto read =
RawRead(fixture, Address(fixture, Value(&base), Value(0u), 2), Value(0u),
memory, 2);
fixture.Emit(ValueOpcode::ReferenceU32, {read}, 0, 2);
fixture.Plan();
Check(fixture.program.srt_reads.empty() &&
read.ResolveInstruction()->GetOpcode() ==
@@ -307,7 +314,7 @@ void TestRuntime64BitDescriptorOps() {
fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(0u)});
const auto high =
fixture.Emit(ValueOpcode::CompositeExtractU64, {combined, Value(1u)});
fixture.Plan();
fixture.program.descriptor_sources.push_back(
{.dwords = {low, high}, .dword_count = 2});
@@ -374,7 +381,6 @@ void TestUniformFirstLaneSamplerLod() {
{packed, active});
const auto divergent = fixture.Emit(
ValueOpcode::ReadFirstLane, {stale, active});
fixture.Plan();
Check(ValidateRuntimeValue(fixture.program, first),
"uniform sampler LOD construction was rejected");
@@ -396,6 +402,49 @@ void TestUniformFirstLaneSamplerLod() {
"uniform sampler LOD clamp evaluated incorrectly");
}
void TestFloatComparisonDescriptorInputs() {
struct Input {
uint32_t bits;
uint32_t less_equal;
uint32_t greater_equal;
};
constexpr std::array inputs = {
Input{0x00000001u, 0, 1}, Input{0x80000001u, 1, 0},
Input{0x007fffffu, 0, 1}, Input{0x807fffffu, 1, 0},
Input{0x00800000u, 0, 1}, Input{0x80800000u, 1, 0},
Input{0x00000000u, 1, 1}, Input{0x80000000u, 1, 1},
Input{0x7fc00000u, 0, 0}};
for (const bool flush : {false, true}) {
Fixture fixture;
const auto user = fixture.Emit(
ValueOpcode::GetUserData, {Value(static_cast<ScalarReg>(2))});
const auto value = fixture.Emit(ValueOpcode::BitCastF32U32, {user});
const auto less_equal = fixture.Emit(ValueOpcode::FPOrdLessThanEqual32,
{value, Value::F32(0.0f)});
const auto greater_equal = fixture.Emit(ValueOpcode::FPOrdGreaterThanEqual32,
{value, Value::F32(0.0f)});
less_equal.Instruction()->SetFlags(FPCompareFlags{flush});
greater_equal.Instruction()->SetFlags(FPCompareFlags{flush});
const auto low = fixture.Emit(ValueOpcode::SelectU32,
{less_equal, Value(1u), Value(0u)});
const auto high = fixture.Emit(ValueOpcode::SelectU32,
{greater_equal, Value(1u), Value(0u)});
fixture.program.descriptor_sources.push_back(
{.dwords = {low, high}, .dword_count = 2});
for (size_t index = 0; index < inputs.size(); index++) {
const auto &input = inputs[index];
const std::array user_data{input.bits};
DescriptorValue result;
Check(SrtWalker(fixture.program, {.user_data = user_data})
.EvaluateDescriptor(0, result) &&
result.dwords[0] == (flush && index < 4 ? 1 : input.less_equal) &&
result.dwords[1] == (flush && index < 4 ? 1 : input.greater_equal),
"descriptor comparison disagrees with native FP32 input mode");
}
}
}
void TestSharedIntegerRuntimeDependencies() {
Fixture fixture;
const auto lane = fixture.Emit(ValueOpcode::LaneId);
@@ -439,37 +488,34 @@ void TestSharedIntegerRuntimeDependencies() {
}
void TestConstantBufferBounds() {
Fixture fixture;
const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer);
const auto buffer =
fixture.Emit(ValueOpcode::GetBufferResource,
{Value(0x3000u), Value(0u), Value(16u), Value(0u)});
const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer,
{buffer, Value(12u)}, memory);
fixture.Emit(ValueOpcode::GetBufferResource,
{read, Value(0u), Value(16u), Value(0u)});
fixture.Plan();
struct Case {
uint32_t offset;
uint32_t immediate;
bool valid;
uint32_t expected;
};
for (const auto &test : {Case{12u, 0u, true, 0xa5a5a5a5u},
Case{3u, 1u, true, 0x12345678u},
Case{0xfffffffcu, 4u, false, 0u},
Case{16u, 0u, false, 0u}}) {
Fixture fixture;
const auto memory = fixture.AddMemory(ResourceKind::ScalarBuffer, test.immediate);
const auto buffer =
fixture.Emit(ValueOpcode::GetBufferResource,
{Value(0x3000u), Value(0u), Value(16u), Value(0u)});
const auto read = fixture.EmitMemory(ValueOpcode::ReadConstBuffer,
{buffer, Value(test.offset)}, memory);
LoadBuffer(fixture, {read, Value(0u), Value(16u), Value(0u)});
fixture.Plan();
TestMemory memory_image{{{0x300cu, 0xa5a5a5a5u}}};
SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
std::vector<uint32_t> flat;
Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) &&
flat == std::vector<uint32_t>{0xa5a5a5a5u},
"constant-buffer SRT walk failed");
Fixture overflow;
const auto overflow_memory = overflow.AddMemory(ResourceKind::ScalarBuffer);
const auto overflow_buffer =
overflow.Emit(ValueOpcode::GetBufferResource,
{Value(0x3000u), Value(0u), Value(16u), Value(0u)});
const auto overflow_read =
overflow.EmitMemory(ValueOpcode::ReadConstBuffer,
{overflow_buffer, Value(16u)}, overflow_memory);
overflow.Emit(ValueOpcode::GetBufferResource,
{overflow_read, Value(0u), Value(16u), Value(0u)});
overflow.Plan();
Check(!SrtWalker(overflow.program, runtime).RefreshFlatBuffer(flat),
"out-of-bounds constant-buffer walk was accepted");
TestMemory memory_image{{{0x3000u, 0x12345678u}, {0x300cu, 0xa5a5a5a5u}}};
SrtRuntime runtime{.read_memory = ReadMemory, .userdata = &memory_image};
std::vector<uint32_t> flat;
Check(SrtWalker(fixture.program, runtime).RefreshFlatBuffer(flat) == test.valid,
"constant-buffer walk misaligned or wrapped its offset components");
Check(test.valid ? flat == std::vector<uint32_t>{test.expected} : memory_image.reads == 0,
"constant-buffer walk read the wrong word or accessed an out-of-bounds address");
}
}
void TestReadLaneElimination() {
@@ -552,7 +598,7 @@ void TestControlFlowValueSurvivesReadLaneFolding() {
void TestUndefinedRuntimeValueFails() {
Fixture fixture;
const auto undef = fixture.Emit(ValueOpcode::UndefU32);
fixture.Plan();
fixture.program.descriptor_sources.push_back(
{.dwords = {undef}, .dword_count = 1});
DescriptorValue result;
@@ -591,6 +637,7 @@ int main() {
TestControlDependentStandaloneLoadStaysTyped();
TestRuntime64BitDescriptorOps();
TestUniformFirstLaneSamplerLod();
TestFloatComparisonDescriptorInputs();
TestSharedIntegerRuntimeDependencies();
TestConstantBufferBounds();
TestReadLaneElimination();
@@ -611,6 +658,7 @@ int main() {
#include "../src/graphics/shader/recompiler/ir/Program.cpp"
#include "../src/graphics/shader/recompiler/ir/Type.cpp"
#include "../src/graphics/shader/recompiler/ir/Value.cpp"
#include "../src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
#include "../src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp"
#include "../src/graphics/shader/recompiler/ir/passes/ConstantPropagation.cpp"
#include "../src/graphics/shader/recompiler/ir/passes/DeadCodeElimination.cpp"
File diff suppressed because it is too large Load Diff
+511 -231
View File
File diff suppressed because it is too large Load Diff