raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-20 10:56:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

graphics: use per-invocation shader masks

Browse Source
This commit is contained in:
nmzik
2026-08-21 08:02:21 +02:00
parent 79443aa104
commit 49ea1680f8
30 changed files with 897 additions and 1140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp
-58
View File
@@ -1,58 +0,0 @@
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
namespace Libs::Graphics {
ShaderSubgroupCapabilities::ShaderSubgroupCapabilities(const GraphicContext& graphics)
: subgroup_size(graphics.subgroup_size), min_subgroup_size(graphics.min_subgroup_size),
max_subgroup_size(graphics.max_subgroup_size),
required_subgroup_size_stages(graphics.required_subgroup_size_stages),
subgroup_size_control_enabled(graphics.subgroup_size_control_enabled) {}
ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size) {
// A 64-wide guest wave can't assume the host GPU's lane layout matches
// GCN's. AMD RDNA also reports a 64-wide subgroup but uses a different
// lane/pixel layout, which corrupted rendering under "native wave" mode.
// Per-invocation emulation works regardless of the host's lane layout.
if (guest_wave_size == 64u) {
return ShaderLaneMaskMode::PerInvocation;
}
return ShaderLaneMaskMode::NativeWave;
}
ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
vk::ShaderStageFlagBits stage,
const ShaderRecompiler::IR::Program& program) {
const auto guest_wave_size = program.wave_size;
if ((guest_wave_size != 32u && guest_wave_size != 64u) ||
!program.spirv_requirements.has_value()) {
return {};
}
if (stage != vk::ShaderStageFlagBits::eCompute) {
const auto expected = SelectGraphicsLaneMaskMode(guest_wave_size);
if (program.lane_mask_mode != expected || (capabilities.subgroup_size != guest_wave_size &&
expected != ShaderLaneMaskMode::PerInvocation)) {
return {};
}
return {expected == ShaderLaneMaskMode::NativeWave
? ShaderSubgroupMode::Natural
: ShaderSubgroupMode::PerInvocationGraphics,
0};
}
if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) {
return {};
}
if (capabilities.subgroup_size == guest_wave_size) {
return {ShaderSubgroupMode::Natural, 0};
}
if (capabilities.subgroup_size_control_enabled &&
(capabilities.required_subgroup_size_stages & stage) &&
guest_wave_size >= capabilities.min_subgroup_size &&
guest_wave_size <= capabilities.max_subgroup_size) {
return {ShaderSubgroupMode::Controlled, guest_wave_size};
}
return {ShaderSubgroupMode::FlattenedMasks, 0};
}
} // namespace Libs::Graphics
src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h
-40
View File
@@ -1,40 +0,0 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
namespace Libs::Graphics {
enum class ShaderSubgroupMode {
Natural,
Controlled,
PerInvocationGraphics,
FlattenedMasks,
Unsupported
};
struct ShaderSubgroupConfiguration {
ShaderSubgroupMode mode = ShaderSubgroupMode::Unsupported;
uint32_t required_size = 0;
};
struct ShaderSubgroupCapabilities {
explicit ShaderSubgroupCapabilities(const GraphicContext& graphics);
uint32_t subgroup_size = 0;
uint32_t min_subgroup_size = 0;
uint32_t max_subgroup_size = 0;
vk::ShaderStageFlags required_subgroup_size_stages;
bool subgroup_size_control_enabled = false;
};
ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size);
ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
vk::ShaderStageFlagBits stage,
const ShaderRecompiler::IR::Program& program);
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
src/graphics/host_gpu/renderer/pipeline/shaders.cpp
+11 -59
View File
@@ -8,7 +8,6 @@
#include "graphics/host_gpu/renderer/debug.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/renderer/renderTarget.h"
@@ -415,48 +414,6 @@ static void CreateLayout(DescriptorCache& descriptor_cache,
}
}
static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
const ShaderRecompiler::IR::Program& program,
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
vk::PipelineShaderStageCreateInfo& stage) {
const auto config =
ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program);
switch (config.mode) {
case ShaderSubgroupMode::Natural: return;
case ShaderSubgroupMode::PerInvocationGraphics: {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation "
"EXEC/VCC on the host-native subgroup\n",
program.wave_size, static_cast<uint32_t>(vk_stage));
}
return;
}
case ShaderSubgroupMode::FlattenedMasks: {
static std::atomic<uint32_t> log_count {0};
if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation "
"EXEC/VCC\n",
program.wave_size, static_cast<uint32_t>(vk_stage));
}
return;
}
case ShaderSubgroupMode::Controlled: break;
case ShaderSubgroupMode::Unsupported:
default:
EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u "
"controlled_stages=0x%08x\n",
program.wave_size, static_cast<uint32_t>(vk_stage), graphics.subgroup_size,
graphics.min_subgroup_size, graphics.max_subgroup_size,
static_cast<vk::ShaderStageFlags::MaskType>(
graphics.required_subgroup_size_stages));
}
required.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
required.requiredSubgroupSize = config.required_size;
stage.pNext = &required;
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
void CreatePipelineInternal(
GraphicContext& graphics, DescriptorCache& descriptor_cache,
@@ -528,8 +485,7 @@ void CreatePipelineInternal(
rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {};
vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
vert_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
vert_shader_stage_info.pNext = nullptr;
@@ -538,12 +494,8 @@ void CreatePipelineInternal(
vert_shader_stage_info.module = vert_shader_module;
vert_shader_stage_info.pName = "main";
vert_shader_stage_info.pSpecializationInfo = nullptr;
EXIT_IF(!vs_input_info.stage);
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
vert_subgroup_size, vert_shader_stage_info);
vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {};
vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
frag_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
frag_shader_stage_info.pNext = nullptr;
frag_shader_stage_info.flags = {};
@@ -551,12 +503,6 @@ void CreatePipelineInternal(
frag_shader_stage_info.module = frag_shader_module;
frag_shader_stage_info.pName = "main";
frag_shader_stage_info.pSpecializationInfo = nullptr;
if (ps_active) {
EXIT_IF(!ps_input_info->stage);
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
*ps_input_info->stage.program, frag_subgroup_size,
frag_shader_stage_info);
}
vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
tess_control_shader_stage_info.sType = vk::StructureType::ePipelineShaderStageCreateInfo;
@@ -1074,8 +1020,15 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
comp_shader_stage_info.pName = "main";
comp_shader_stage_info.pSpecializationInfo = nullptr;
EXIT_IF(!input_info.stage);
ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
comp_subgroup_size, comp_shader_stage_info);
const auto wave_size = input_info.stage.program->wave_size;
if (graphics.subgroup_size_control_enabled &&
(graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) {
comp_subgroup_size.sType =
vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
comp_subgroup_size.requiredSubgroupSize = wave_size;
comp_shader_stage_info.pNext = &comp_subgroup_size;
}
vk::DescriptorSetLayout set_layouts[1] = {};
uint32_t set_layouts_num = 0;
@@ -1083,7 +1036,6 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
vk::PushConstantRange push_constant_info[1];
uint32_t push_constant_info_num = 0;
EXIT_IF(!input_info.stage);
CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
push_constant_info_num, *input_info.stage.program,
vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);
src/graphics/host_gpu/renderer/renderCompute.cpp
-1
View File
@@ -15,7 +15,6 @@
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vulkanCommon.h"
src/graphics/host_gpu/renderer/renderDraw.cpp
+2 -5
View File
@@ -18,7 +18,6 @@
#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/host_gpu/vulkanCommon.h"
@@ -1061,12 +1060,10 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
for (uint32_t i = 0; i < state.color_count; i++) {
target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
}
const auto lane_mask_mode = SelectGraphicsLaneMaskMode(64u);
if (log_phases) {
LogDrawPhase(draw.name, "ShaderCompileInfoVS");
}
if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, state.vs_input_info,
state.vs_shader)) {
EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name);
}
@@ -1077,7 +1074,7 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
if (log_phases) {
LogDrawPhase(draw.name, "ShaderCompileInfoPS");
}
if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, state.vs_input_info,
target_export_mapping, state.ps_input_info, state.ps_shader)) {
EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name);
}
src/graphics/presentation/window/vulkanWindow.cpp
+8 -17
View File
@@ -514,10 +514,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
features12.pNext = &depth_clip_control;
features12.samplerMirrorClampToEdge = VK_TRUE;
vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {};
subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures;
subgroup_size_control.pNext = &features12;
vk::PhysicalDeviceVulkan13Features supported_features13 {};
supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
@@ -578,28 +574,23 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
device_features.shaderInt64 = supported_features2.features.shaderInt64;
}
if (graphics.subgroup_size_control_enabled &&
supported_features13.subgroupSizeControl == VK_TRUE) {
subgroup_size_control.subgroupSizeControl = VK_TRUE;
}
const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE
? static_cast<const void*>(&subgroup_size_control)
: static_cast<const void*>(&features12));
vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
robustness2.pNext = const_cast<void*>(base_feature_chain);
robustness2.pNext = &features12;
if (robustness2_ext_enabled) {
robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2;
robustness2.robustImageAccess2 = supported_robustness2.robustImageAccess2;
robustness2.nullDescriptor = supported_robustness2.nullDescriptor;
}
auto features13 = required_features13;
features13.pNext =
robustness2_ext_enabled ? &robustness2 : const_cast<void*>(base_feature_chain);
const bool subgroup_size_control_enabled =
graphics.subgroup_size_control_enabled && supported_features13.subgroupSizeControl == VK_TRUE;
auto features13 = required_features13;
features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
: static_cast<void*>(&features12);
features13.robustImageAccess = supported_features13.robustImageAccess;
features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
features13.robustImageAccess == VK_TRUE ? "true" : "false",
src/graphics/shader/recompiler/ShaderRecompiler.cpp
-1
View File
@@ -683,7 +683,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
return false;
}
output.lane_mask_mode = options.lane_mask_mode;
output.shader_hash = options.shader_hash;
output.user_data_base = options.user_data_base;
output.user_data_count = options.user_data_count;
src/graphics/shader/recompiler/ShaderRecompiler.h
-1
View File
@@ -14,7 +14,6 @@ namespace Libs::Graphics::ShaderRecompiler {
struct CompileOptions {
ShaderType stage = ShaderType::Compute;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+9 -14
View File
@@ -198,10 +198,7 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
program.spirv_requirements.emplace(requirements);
return true;
}
const auto MarkExactSubgroup = [&] {
requirements.requires_exact_subgroup = true;
requirements.subgroup_ballot = true;
};
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
for (const auto* block: program.values->blocks) {
for (const auto& inst: *block) {
if (IR::AddressOpcodeInfoOf(inst.GetOpcode()).access != IR::AddressAccess::None) {
@@ -230,7 +227,6 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
if (program.stage != ShaderType::Compute) {
return Fail(error, "buffer ADD_TID is only valid for compute shaders");
}
requirements.requires_exact_subgroup = true;
requirements.subgroup_local_invocation_id = true;
}
}
@@ -250,17 +246,17 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
}
if (shared_access == IR::SharedAccess::Append ||
shared_access == IR::SharedAccess::Consume) {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
}
}
switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot: MarkExactSubgroup(); break;
case IR::ValueOpcode::Ballot: MarkBallot(); break;
case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::ReadFirstLane:
case IR::ValueOpcode::ReadLane: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
requirements.subgroup_local_invocation_id = true;
@@ -270,18 +266,18 @@ bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WqmMask:
case IR::ValueOpcode::WriteLane: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::Permlane16U32: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::SwizzleU32: {
MarkExactSubgroup();
MarkBallot();
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
break;
@@ -346,9 +342,8 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
return false;
}
EmitterState state(program, resources, input_info);
state.stage = program.stage;
state.wave_size = program.wave_size;
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
state.stage = program.stage;
state.wave_size = program.wave_size;
state.inputs.reserve(program.info.inputs.size());
state.outputs.reserve(program.info.outputs.size());
state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+4 -19
View File
@@ -58,13 +58,6 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
return result;
}
uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), result, value,
ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
return result;
}
uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
auto& state = ctx.state;
if (kind == IR::StageInputKind::LocalInvocationIndex) {
@@ -222,8 +215,8 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
bool uint_output) {
auto& state = ctx.state;
if (exp.compr && !uint_output) {
const auto unpack = MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16
: GlslUnpackHalf2x16;
const auto unpack =
MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
ConstantF32(state, 0x3f800000u)};
for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -441,16 +434,8 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
return true;
case IR::ValueOpcode::Ballot:
if (state.per_invocation_masks) {
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result,
BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
ConstantU32(state, 0), ConstantU32(state, 0)});
ctx.Define(inst, result);
} else {
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
}
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
return true;
case IR::ValueOpcode::ReadFirstLane: {
const auto ballot = state.builder.AllocateId();
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+53 -54
View File
@@ -251,30 +251,30 @@ enum : uint32_t {
};
enum : uint32_t {
GlslRoundEven = 2,
GlslTrunc = 3,
GlslFAbs = 4,
GlslFloor = 8,
GlslCeil = 9,
GlslFract = 10,
GlslSin = 13,
GlslCos = 14,
GlslExp2 = 29,
GlslLog2 = 30,
GlslSqrt = 31,
GlslInverseSqrt = 32,
GlslFMin = 37,
GlslFMax = 40,
GlslFClamp = 43,
GlslLdexp = 53,
GlslFma = 50,
GlslPackSnorm2x16 = 56,
GlslPackUnorm2x16 = 57,
GlslPackHalf2x16 = 58,
GlslRoundEven = 2,
GlslTrunc = 3,
GlslFAbs = 4,
GlslFloor = 8,
GlslCeil = 9,
GlslFract = 10,
GlslSin = 13,
GlslCos = 14,
GlslExp2 = 29,
GlslLog2 = 30,
GlslSqrt = 31,
GlslInverseSqrt = 32,
GlslFMin = 37,
GlslFMax = 40,
GlslFClamp = 43,
GlslLdexp = 53,
GlslFma = 50,
GlslPackSnorm2x16 = 56,
GlslPackUnorm2x16 = 57,
GlslPackHalf2x16 = 58,
GlslUnpackUnorm2x16 = 61,
GlslUnpackHalf2x16 = 62,
GlslFindILsb = 73,
GlslFindUMsb = 75,
GlslUnpackHalf2x16 = 62,
GlslFindILsb = 73,
GlslFindUMsb = 75,
};
struct InputBinding {
@@ -325,41 +325,40 @@ struct EmitterState {
: program(program_), resources(resources_), input_info(input_info_),
requirements(*program_.spirv_requirements) {}
Builder builder;
const IR::Program& program;
const IR::ResourceSnapshot& resources;
ShaderStageInputInfo input_info;
const IR::SpirvRequirements& requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
bool per_invocation_masks = false;
uint32_t storage_buffer_variable = 0;
Builder builder;
const IR::Program& program;
const IR::ResourceSnapshot& resources;
ShaderStageInputInfo input_info;
const IR::SpirvRequirements& requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t wave_size = 64;
uint32_t storage_buffer_variable = 0;
std::array<uint32_t, IR::ShaderInfo::MaxBuffers + IR::ShaderInfo::MaxAddresses>
memory_byte_offsets {};
uint32_t address_memory_variable = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
uint32_t vsharp_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
memory_byte_offsets {};
uint32_t address_memory_variable = 0;
uint32_t gds_variable = 0;
uint32_t gds_length = 0;
uint32_t push_constant_variable = 0;
uint32_t vsharp_storage_variable = 0;
uint32_t flattened_srt_variable = 0;
uint32_t lds_variable = 0;
uint32_t scratch_variable = 0;
std::array<uint32_t, 2u * SampledImageViewKindCount> sampled_image_variables {};
std::array<uint32_t,
static_cast<uint32_t>(StorageImageClass::Count) * StorageImageViewKindCount>
storage_image_variables {};
uint32_t sampler_variable = 0;
uint32_t main_func = 0;
uint32_t entry_label = 0;
uint32_t current_label = 0;
uint32_t pixel_valid_mask_variable = 0;
uint32_t subgroup_local_invocation_id_variable = 0;
uint32_t per_vertex_variable = 0;
uint32_t depth_variable = 0;
uint32_t sample_mask_variable = 0;
std::vector<InputBinding> inputs;
std::vector<OutputBinding> outputs;
std::vector<uint32_t> interface_variables;
storage_image_variables {};
uint32_t sampler_variable = 0;
uint32_t main_func = 0;
uint32_t entry_label = 0;
uint32_t current_label = 0;
uint32_t pixel_valid_mask_variable = 0;
uint32_t subgroup_local_invocation_id_variable = 0;
uint32_t per_vertex_variable = 0;
uint32_t depth_variable = 0;
uint32_t sample_mask_variable = 0;
std::vector<InputBinding> inputs;
std::vector<OutputBinding> outputs;
std::vector<uint32_t> interface_variables;
};
uint32_t TypeVoid(EmitterState& state);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+2 -4
View File
@@ -672,11 +672,9 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 2);
const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 3);
const auto count =
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), count_low),
Unary(state, OpBitCount, TypeU32(state), count_high));
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low),
Unary(state, OpBitCount, TypeU32(state), high));
const auto first = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), first,
ConstantU32(state, ScopeSubgroup), ballot});
src/graphics/shader/recompiler/frontend/translate/Control.cpp
+66 -40
View File
@@ -7,37 +7,26 @@ bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::SaveexecB32:
case IR::Opcode::SaveexecB64: {
const auto old_low = ir.GetExecLo();
const auto old_high = ir.GetExecHi();
const auto src_low = ReadRawU32(inst.src[0]);
const auto src_high = inst.op == IR::Opcode::SaveexecB64
? ReadRawU32(OffsetOperand(inst.src[0], 1))
: IR::U32(IR::Value(0u));
const auto apply = [&](IR::U32 old_value, IR::U32 src_value) {
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value);
case IR::SaveexecMode::Andn1:
return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value));
case IR::SaveexecMode::Orn2:
return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value);
}
return old_value;
};
const auto result_low = apply(old_low, src_low);
const auto result_high =
inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
if (inst.op == IR::Opcode::SaveexecB64) {
WriteU32Pair(inst.dst, {old_low, old_high});
} else {
WriteOperand(inst.dst, old_low);
const auto old = ir.GetExec();
const auto src = ReadMask(inst.src[0]);
IR::U1 result;
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: result = ir.LogicalAnd(old, src); break;
case IR::SaveexecMode::Andn1:
result = ir.LogicalAnd(old, ir.LogicalNot(src));
break;
case IR::SaveexecMode::Orn2: result = ir.LogicalOr(ir.LogicalNot(old), src); break;
}
ir.SetExecLo(result_low);
ir.SetExecHi(result_high);
ir.SetExec(ThreadBit(result_low, result_high));
const auto nonzero = inst.op == IR::Opcode::SaveexecB64
? ir.BitwiseOr(result_low, result_high)
: result_low;
ir.SetScc(ir.INotEqual(nonzero, zero));
if (inst.op == IR::Opcode::SaveexecB64) {
WriteMask64(inst.dst, old);
} else {
WriteMask(inst.dst, old);
}
const auto mask = BallotMask(result);
ir.SetExec(result);
ir.SetExecLo(mask[0]);
ir.SetExecHi(mask[1]);
ir.SetScc(result);
return true;
}
case IR::Opcode::ScalarAddCarryU32:
@@ -157,24 +146,61 @@ bool Translator::TranslateMove(const IR::Instruction& inst) {
WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
return true;
case IR::Opcode::MoveU64: {
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0])) {
WriteMask64(inst.dst, ReadMask(inst.src[0]));
return true;
}
const bool scalar_copy = inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar;
IR::U1 source_mask;
IR::U1 source_mask_valid;
if (scalar_copy) {
const auto src = static_cast<IR::ScalarReg>(inst.src[0].reg.index);
source_mask = ir.GetThreadBitScalarReg(src);
source_mask_valid = ir.GetScalarMaskTag(src);
}
WriteU32Pair(inst.dst, ReadU32Pair(inst.src[0]));
if (inst.dst.kind == IR::OperandKind::Register &&
inst.src[0].kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar &&
inst.src[0].reg.file == IR::RegisterFile::Scalar) {
ir.SetThreadBitScalarReg(
static_cast<IR::ScalarReg>(inst.dst.reg.index),
ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
if (scalar_copy) {
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.dst.reg.index),
source_mask);
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(inst.dst.reg.index),
source_mask_valid);
}
return true;
}
case IR::Opcode::WqmB64: {
if (!current_per_invocation_masks) {
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (!is_exec_or_vcc(inst.dst) && !is_exec_or_vcc(inst.src[0])) {
const auto mask_valid = ReadMaskValid(inst.src[0]);
const auto invocation_result =
IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
const auto result = IR::U64(
ir.Emit(IR::ValueOpcode::WqmU64, {ReadOperand(inst.src[0], IR::Type::U64)}));
WriteOperand(inst.dst, result);
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero = IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})}));
ir.SetScc(IR::U1(ir.Emit(
IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
}
return true;
}
const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp
+69 -6
View File
@@ -3,13 +3,76 @@
namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
if (current_per_invocation_masks) {
return TranslatePerInvocationU64Mask(inst);
const auto invocation_result = EvaluateU64Mask(inst);
const auto is_exec_or_vcc = [](const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc);
};
if (is_exec_or_vcc(inst.dst) || is_exec_or_vcc(inst.src[0]) ||
(inst.src_count > 1u && is_exec_or_vcc(inst.src[1]))) {
WriteMask64(inst.dst, invocation_result);
ir.SetScc(invocation_result);
return true;
}
return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64
? TranslateSimpleInteger(inst)
: TranslateComposedInteger(inst);
const auto lhs = ReadU32Pair(inst.src[0]);
auto mask_valid = ReadMaskValid(inst.src[0]);
if (inst.src_count > 1u) {
mask_valid = ir.LogicalAnd(mask_valid, ReadMaskValid(inst.src[1]));
}
std::array<IR::U32, 2> result;
if (inst.op == IR::Opcode::BitwiseNotU64) {
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
} else {
const auto rhs = ReadU32Pair(inst.src[1]);
for (uint32_t component = 0; component < 2u; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndU64:
result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseAndNotU64:
result[component] =
ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseOrU64:
result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseOrNotU64:
result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseXorU64:
result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseNandU64:
result[component] =
ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseNorU64:
result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseXnorU64:
result[component] =
ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
break;
default: return false;
}
}
}
WriteU32Pair(inst.dst, result);
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, invocation_result);
ir.SetScalarMaskTag(dst, mask_valid);
const auto raw_nonzero =
ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u)));
ir.SetScc(IR::U1(
ir.Emit(IR::ValueOpcode::SelectU1, {mask_valid, invocation_result, raw_nonzero})));
} else {
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
}
return true;
}
bool Translator::TranslateInstruction(const IR::Instruction& inst) {
src/graphics/shader/recompiler/frontend/translate/Integer.cpp
+31 -80
View File
@@ -126,62 +126,26 @@ bool Translator::TranslatePackedInteger16(const IR::Instruction& inst) {
}
}
bool Translator::TranslatePerInvocationU64Mask(const IR::Instruction& inst) {
if (!current_per_invocation_masks) {
return false;
}
IR::U1 Translator::EvaluateU64Mask(const IR::Instruction& inst) {
const auto lhs = [&] { return ReadMask(inst.src[0]); };
const auto rhs = [&] { return ReadMask(inst.src[1]); };
IR::U1 result;
switch (inst.op) {
case IR::Opcode::BitwiseAndU64: result = ir.LogicalAnd(lhs(), rhs()); break;
case IR::Opcode::BitwiseOrU64: result = ir.LogicalOr(lhs(), rhs()); break;
case IR::Opcode::BitwiseAndU64: return ir.LogicalAnd(lhs(), rhs());
case IR::Opcode::BitwiseOrU64: return ir.LogicalOr(lhs(), rhs());
case IR::Opcode::BitwiseXorU64:
result = IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
break;
case IR::Opcode::BitwiseAndNotU64:
result = ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
break;
case IR::Opcode::BitwiseOrNotU64: result = ir.LogicalOr(lhs(), ir.LogicalNot(rhs())); break;
case IR::Opcode::BitwiseNandU64: result = ir.LogicalNot(ir.LogicalAnd(lhs(), rhs())); break;
case IR::Opcode::BitwiseNorU64: result = ir.LogicalNot(ir.LogicalOr(lhs(), rhs())); break;
return IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()}));
case IR::Opcode::BitwiseAndNotU64: return ir.LogicalAnd(lhs(), ir.LogicalNot(rhs()));
case IR::Opcode::BitwiseOrNotU64: return ir.LogicalOr(lhs(), ir.LogicalNot(rhs()));
case IR::Opcode::BitwiseNandU64: return ir.LogicalNot(ir.LogicalAnd(lhs(), rhs()));
case IR::Opcode::BitwiseNorU64: return ir.LogicalNot(ir.LogicalOr(lhs(), rhs()));
case IR::Opcode::BitwiseXnorU64:
result = ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
break;
case IR::Opcode::BitwiseNotU64: result = ir.LogicalNot(lhs()); break;
default: return false;
return ir.LogicalNot(IR::U1(ir.Emit(IR::ValueOpcode::LogicalXor, {lhs(), rhs()})));
case IR::Opcode::BitwiseNotU64: return ir.LogicalNot(lhs());
default: EXIT("invalid 64-bit mask opcode");
}
WriteMask64(inst.dst, result);
return true;
}
bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64 ||
inst.op == IR::Opcode::BitwiseXorU64 || inst.op == IR::Opcode::BitwiseNotU64) {
const auto lhs = ReadU32Pair(inst.src[0]);
std::array<IR::U32, 2> result;
if (inst.op == IR::Opcode::BitwiseNotU64) {
result = {ir.BitwiseNot(lhs[0]), ir.BitwiseNot(lhs[1])};
} else {
const auto rhs = ReadU32Pair(inst.src[1]);
for (uint32_t component = 0; component < 2; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndU64:
result[component] = ir.BitwiseAnd(lhs[component], rhs[component]);
break;
case IR::Opcode::BitwiseOrU64:
result[component] = ir.BitwiseOr(lhs[component], rhs[component]);
break;
default:
result[component] = ir.BitwiseXor(lhs[component], rhs[component]);
break;
}
}
}
WriteU32Pair(inst.dst, result);
return true;
}
IR::ValueOpcode opcode {};
IR::Type type = IR::Type::U32;
switch (inst.op) {
@@ -250,37 +214,6 @@ bool Translator::TranslateSimpleInteger(const IR::Instruction& inst) {
}
bool Translator::TranslateComposedInteger(const IR::Instruction& inst) {
if (inst.op == IR::Opcode::BitwiseAndNotU64 || inst.op == IR::Opcode::BitwiseOrNotU64 ||
inst.op == IR::Opcode::BitwiseNandU64 || inst.op == IR::Opcode::BitwiseNorU64 ||
inst.op == IR::Opcode::BitwiseXnorU64) {
const auto lhs = ReadU32Pair(inst.src[0]);
const auto rhs = ReadU32Pair(inst.src[1]);
std::array<IR::U32, 2> result;
for (uint32_t component = 0; component < 2; component++) {
switch (inst.op) {
case IR::Opcode::BitwiseAndNotU64:
result[component] =
ir.BitwiseAnd(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseOrNotU64:
result[component] = ir.BitwiseOr(lhs[component], ir.BitwiseNot(rhs[component]));
break;
case IR::Opcode::BitwiseNandU64:
result[component] =
ir.BitwiseNot(ir.BitwiseAnd(lhs[component], rhs[component]));
break;
case IR::Opcode::BitwiseNorU64:
result[component] = ir.BitwiseNot(ir.BitwiseOr(lhs[component], rhs[component]));
break;
default:
result[component] =
ir.BitwiseNot(ir.BitwiseXor(lhs[component], rhs[component]));
break;
}
}
WriteU32Pair(inst.dst, result);
return true;
}
const auto binary_u32 = [&](auto operation) {
const auto lhs = ReadU32(inst.src[0]);
const auto rhs = ReadU32(inst.src[1]);
@@ -612,10 +545,28 @@ bool Translator::TranslateExtendedInteger(const IR::Instruction& inst) {
}
case IR::Opcode::SelectU64: {
const auto condition = ReadCondition(inst.src[0]);
const auto lhs = ReadU32Pair(inst.src[1]);
const auto rhs = ReadU32Pair(inst.src[2]);
const auto selected_mask =
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1,
{condition, ReadMask(inst.src[1]), ReadMask(inst.src[2])}));
const auto selected_mask_valid =
IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {condition, ReadMaskValid(inst.src[1]),
ReadMaskValid(inst.src[2])}));
if (inst.dst.kind == IR::OperandKind::Register &&
(inst.dst.reg.file == IR::RegisterFile::Exec ||
inst.dst.reg.file == IR::RegisterFile::Vcc)) {
WriteMask64(inst.dst, selected_mask);
return true;
}
const auto lhs = ReadU32Pair(inst.src[1]);
const auto rhs = ReadU32Pair(inst.src[2]);
WriteU32Pair(inst.dst, {ir.Select(condition, lhs[0], rhs[0]),
ir.Select(condition, lhs[1], rhs[1])});
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg.index);
ir.SetThreadBitScalarReg(dst, selected_mask);
ir.SetScalarMaskTag(dst, selected_mask_valid);
}
return true;
}
case IR::Opcode::PackLowLowU16:
src/graphics/shader/recompiler/frontend/translate/Translate.cpp
+80 -80
View File
@@ -91,19 +91,8 @@ IR::U32 Translator::ReadPcRelativeU32(const IR::Operand& operand) {
}
std::array<IR::U32, 2> Translator::BallotMask(IR::U1 value) {
if (current_per_invocation_masks) {
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
const auto ballot = ir.Emit(IR::ValueOpcode::Ballot, {value});
const auto low =
IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(0u)}));
// RDNA2 wave32 masks have one architectural dword; the upper half is unused.
const auto high =
current_wave_size == 32u
? IR::U32(IR::Value(0u))
: IR::U32(ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {ballot, IR::Value(1u)}));
return {low, high};
return {ir.Select(value, IR::U32(IR::Value(1u)), IR::U32(IR::Value(0u))),
IR::U32(IR::Value(0u))};
}
IR::U32 Translator::ReadRawU32(const IR::Operand& operand) {
@@ -271,9 +260,17 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
}
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
ir.SetScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), value);
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
ir.SetScalarReg(reg, value);
ir.SetThreadBitScalarReg(reg, ThreadBit(value));
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
break;
}
case IR::RegisterFile::Vector: {
const auto reg = static_cast<IR::VectorReg>(operand.reg.index);
const auto old = ir.GetVectorReg(reg);
@@ -296,20 +293,20 @@ void Translator::WriteRawU32(const IR::Operand& operand, IR::U32 value) {
case IR::RegisterFile::Vcc:
if (operand.reg.index == 0) {
ir.SetVccLo(IR::U32(value));
ir.SetVcc(ThreadBit(IR::U32(value), ir.GetVccHi()));
ir.SetVcc(ThreadBit(IR::U32(value)));
} else {
ir.SetVccHi(IR::U32(value));
ir.SetVcc(ThreadBit(ir.GetVccLo(), IR::U32(value)));
ir.SetVcc(ThreadBit(ir.GetVccLo()));
}
break;
case IR::RegisterFile::M0: ir.SetM0(IR::U32(value)); break;
case IR::RegisterFile::Exec:
if (operand.reg.index == 0) {
ir.SetExecLo(IR::U32(value));
ir.SetExec(ThreadBit(IR::U32(value), ir.GetExecHi()));
ir.SetExec(ThreadBit(IR::U32(value)));
} else {
ir.SetExecHi(IR::U32(value));
ir.SetExec(ThreadBit(ir.GetExecLo(), IR::U32(value)));
ir.SetExec(ThreadBit(ir.GetExecLo()));
}
break;
case IR::RegisterFile::Scc: ir.SetScc(ir.INotEqual(value, IR::U32(IR::Value(0u)))); break;
@@ -459,10 +456,10 @@ IR::U64 Translator::ReadU64(const IR::Operand& operand) {
IR::F32 Translator::ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed) {
if (operand.float_inline) {
const bool use_zero = packed && (high_lane ? operand.op_sel_hi : operand.op_sel);
auto value = use_zero ? IR::F32(IR::Value::F32(0.0f))
: ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
auto value = use_zero ? IR::F32(IR::Value::F32(0.0f))
: ir.BitCastF32(IR::U32(IR::Value(operand.imm)));
const auto half = IR::F16(ir.Emit(IR::ValueOpcode::ConvertF16F32, {value}));
value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32F16, {half}));
if (operand.absolute) {
value = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {value}));
}
@@ -561,7 +558,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
return;
}
if (operand.kind == IR::OperandKind::Register) {
const auto thread_bit = ThreadBit(value[0], value[1]);
const auto thread_bit = ThreadBit(value[0]);
switch (operand.reg.file) {
case IR::RegisterFile::Exec:
ir.SetExec(thread_bit);
@@ -573,9 +570,7 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
ir.SetVccLo(value[0]);
ir.SetVccHi(value[1]);
return;
case IR::RegisterFile::Scalar:
ir.SetThreadBitScalarReg(static_cast<IR::ScalarReg>(operand.reg.index), thread_bit);
break;
case IR::RegisterFile::Scalar: break;
default: break;
}
}
@@ -583,21 +578,8 @@ void Translator::WriteU32Pair(const IR::Operand& operand, const std::array<IR::U
WriteRawU32(OffsetOperand(operand, 1), value[1]);
}
IR::U1 Translator::ThreadBit(IR::U32 low, IR::U32 high) {
if (current_per_invocation_masks) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
auto word = low;
if (current_wave_size == 64u) {
const auto high_lane =
IR::U1(ir.Emit(IR::ValueOpcode::UGreaterThanEqual32, {lane, IR::Value(32u)}));
word = ir.Select(high_lane, high, low);
}
const auto bit =
ir.BitwiseAnd(ir.ShiftRightLogical(word, ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)))),
IR::U32(IR::Value(1u)));
return ir.INotEqual(bit, IR::U32(IR::Value(0u)));
IR::U1 Translator::ThreadBit(IR::U32 low) {
return ir.INotEqual(low, IR::U32(IR::Value(0u)));
}
IR::U1 Translator::ReadCondition(const IR::Operand& operand) {
@@ -619,11 +601,7 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
switch (operand.reg.file) {
case IR::RegisterFile::Scalar: {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto high =
current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs
? ir.GetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u))
: IR::U32(IR::Value(0u));
return ThreadBit(ir.GetScalarReg(reg), high);
return ir.GetThreadBitScalarReg(reg);
}
case IR::RegisterFile::Exec: return ir.GetExec();
case IR::RegisterFile::Vcc: return ir.GetVcc();
@@ -632,6 +610,25 @@ IR::U1 Translator::ReadMask(const IR::Operand& operand) {
}
}
IR::U1 Translator::ReadMaskValid(const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::ImmediateU32) {
const bool zero = operand.imm == 0u && !operand.sext_64;
const bool ones = operand.imm == 0xffffffffu && operand.sext_64;
return IR::U1(IR::Value(zero || ones));
}
if (operand.kind != IR::OperandKind::Register) {
return IR::U1(IR::Value(false));
}
switch (operand.reg.file) {
case IR::RegisterFile::Scalar:
return ir.GetScalarMaskTag(static_cast<IR::ScalarReg>(operand.reg.index));
case IR::RegisterFile::Exec:
case IR::RegisterFile::Vcc:
case IR::RegisterFile::Scc: return IR::U1(IR::Value(true));
default: return IR::U1(IR::Value(false));
}
}
void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
EXIT_IF(operand.kind != IR::OperandKind::Register);
switch (operand.reg.file) {
@@ -639,10 +636,18 @@ void Translator::WriteMask(const IR::Operand& operand, IR::U1 value) {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
if (current_wave_size == 64u && IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
return;
}
@@ -676,9 +681,17 @@ void Translator::WriteMask64(const IR::Operand& operand, IR::U1 value) {
const auto reg = static_cast<IR::ScalarReg>(operand.reg.index);
const auto mask = BallotMask(value);
ir.SetThreadBitScalarReg(reg, value);
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(true)));
if (IR::RegIndex(reg) > 0u) {
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
IR::U1(IR::Value(false)));
}
ir.SetScalarReg(reg, mask[0]);
if (IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
ir.SetScalarReg(static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u), mask[1]);
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
ir.SetScalarReg(high, mask[1]);
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
}
}
@@ -733,23 +746,17 @@ bool Translator::AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlock
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
return true;
}
IR::U1 condition;
const auto mask_non_zero = [&](IR::U1 value) {
const auto mask = BallotMask(value);
return ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u)));
};
// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
// invocations leave the region without reconstructing a host-subgroup mask.
IR::U1 condition;
switch (source.terminator.condition) {
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
case CFG::BranchCondition::VccZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetVcc()));
break;
case CFG::BranchCondition::VccNonZero: condition = mask_non_zero(ir.GetVcc()); break;
case CFG::BranchCondition::ExecZero:
condition = ir.LogicalNot(mask_non_zero(ir.GetExec()));
break;
case CFG::BranchCondition::ExecNonZero: condition = mask_non_zero(ir.GetExec()); break;
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
case CFG::BranchCondition::GotoVariable:
if (source.terminator.goto_variable == UINT32_MAX) {
return Fail(error,
@@ -771,11 +778,10 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::string* error) {
result = {};
const uint32_t wave_size = source.wave_size;
const bool per_invocation_masks = source.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
result = {};
const uint32_t wave_size = source.wave_size;
uint32_t vector_limit = 1u;
const auto include_vector = [&](const IR::Operand& operand, uint32_t count = 1u) {
if (operand.kind == IR::OperandKind::Register &&
operand.reg.file == IR::RegisterFile::Vector) {
vector_limit =
@@ -852,19 +858,14 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
if (IR::RegIndex(reg) >= IR::NumScalarRegs) {
break;
}
entry_ir.SetScalarReg(reg, entry_ir.GetUserData(reg));
const auto value = entry_ir.GetUserData(reg);
entry_ir.SetScalarReg(reg, value);
entry_ir.SetThreadBitScalarReg(reg, entry_ir.INotEqual(value, IR::U32(IR::Value(0u))));
entry_ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
}
entry_ir.SetExec(IR::U1(IR::Value(true)));
if (source.stage == ShaderType::Compute && !per_invocation_masks) {
const auto active = entry_ir.Emit(IR::ValueOpcode::Ballot, {IR::Value(true)});
entry_ir.SetExecLo(entry_ir.CompositeExtract(active, 0));
entry_ir.SetExecHi(source.wave_size > 32u ? entry_ir.CompositeExtract(active, 1)
: IR::U32(IR::Value(0u)));
} else {
entry_ir.SetExecLo(IR::U32(IR::Value(per_invocation_masks ? 1u : 0xffffffffu)));
entry_ir.SetExecHi(IR::U32(
IR::Value(!per_invocation_masks && source.wave_size > 32u ? 0xffffffffu : 0u)));
}
entry_ir.SetExecLo(IR::U32(IR::Value(1u)));
entry_ir.SetExecHi(IR::U32(IR::Value(0u)));
if (source.stage == ShaderType::Compute) {
const auto* cs = compute_input_info;
const auto thread_ids =
@@ -930,8 +931,7 @@ bool TranslateProgram(const IR::Program& source, IR::ValueProgram& result,
}
}
for (size_t index = 0; index < source.blocks.size(); index++) {
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size,
per_invocation_masks);
Detail::Translator translator(result, result.blocks[index + 1u], vector_limit, wave_size);
if (!translator.TranslateBlock(source.blocks[index], error) ||
!translator.AddBranchCondition(source.blocks[index], result.block_info[index + 1u],
error)) {
src/graphics/shader/recompiler/frontend/translate/Translator.h
+35 -36
View File
@@ -10,9 +10,9 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
class Translator {
public:
Translator(IR::ValueProgram& program, IR::Block* block, uint32_t vector_limit,
uint32_t wave_size, bool per_invocation_masks)
uint32_t wave_size)
: value_program(program), ir(block), current_vector_limit(vector_limit),
current_wave_size(wave_size), current_per_invocation_masks(per_invocation_masks) {}
current_wave_size(wave_size) {}
bool TranslateBlock(const IR::BasicBlock& source, std::string* error);
bool AddBranchCondition(const IR::BasicBlock& source, IR::ValueBlockInfo& info,
@@ -28,7 +28,7 @@ private:
IR::U32 ReadScalarCode(uint32_t code);
IR::U32 ApplyBitSourceModifiers(const IR::Operand& operand, IR::U32 value);
IR::Value ReadOperand(const IR::Operand& operand, IR::Type type);
IR::U1 ThreadBit(IR::U32 low, IR::U32 high);
IR::U1 ThreadBit(IR::U32 low);
void WriteRawU32(const IR::Operand& operand, IR::U32 value);
IR::F32 ApplyF32ResultModifiers(const IR::Operand& operand, IR::F32 value);
void WriteOperand(const IR::Operand& operand, IR::Value value);
@@ -38,11 +38,10 @@ private:
IR::U32 ReadU32(const IR::Operand& operand);
std::array<IR::U32, 2> ReadU32Pair(const IR::Operand& operand);
IR::U64 ReadU64(const IR::Operand& operand);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane,
bool packed = false);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::F32 ReadF16LaneAsF32(const IR::Operand& operand, bool high_lane, bool packed = false);
IR::F32 ReadF16AsF32(const IR::Operand& operand);
IR::F32 ReadMixF32(const IR::Operand& operand);
IR::U32 ReadU16LaneRaw(const IR::Operand& operand, bool high_lane);
IR::U32 ReadU16LaneAsU32(const IR::Operand& operand, bool high_lane, bool sign_extend);
IR::U32 ReadU16AsU32(const IR::Operand& operand, bool sign_extend);
IR::U32 ReadF16LaneBits(const IR::Operand& operand, bool high_lane);
@@ -51,6 +50,7 @@ private:
IR::U1 ReadCondition(const IR::Operand& operand);
IR::U32 ConditionBit(const IR::Operand& operand);
IR::U1 ReadMask(const IR::Operand& operand);
IR::U1 ReadMaskValid(const IR::Operand& operand);
void WriteMask(const IR::Operand& operand, IR::U1 value);
void WriteMask64(const IR::Operand& operand, IR::U1 value);
void WriteCompareResult(const IR::Operand& operand, IR::U1 value);
@@ -76,8 +76,8 @@ private:
IR::Value GetSamplerResource(const IR::MemoryInfo& memory);
IR::Value MakeImageAddress(const IR::Instruction& inst, const IR::Operand& base);
IR::Value ConstructU32x4(const IR::Operand& base, uint32_t count);
void WriteImageComponents(const IR::Operand& dst, IR::Value value,
const IR::MemoryInfo& memory, uint32_t component_limit);
void WriteImageComponents(const IR::Operand& dst, IR::Value value, const IR::MemoryInfo& memory,
uint32_t component_limit);
IR::ValueOpcode ImageAtomicOpcode(IR::Opcode opcode);
BufferAddress ReadBufferAddress(const IR::Instruction& inst, uint32_t source_offset);
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
@@ -101,35 +101,34 @@ private:
uint32_t upper_result);
IR::U32 PackU16Lanes(IR::U32 low, IR::U32 high);
bool TranslateInstruction(const IR::Instruction& inst);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
bool TranslatePerInvocationU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
bool TranslateInstruction(const IR::Instruction& inst);
bool TranslateStateOperation(const IR::Instruction& inst);
bool TranslateControlOperation(const IR::Instruction& inst);
bool TranslateMove(const IR::Instruction& inst);
bool TranslateLaneOperation(const IR::Instruction& inst);
bool TranslateAttributeOperation(const IR::Instruction& inst);
bool TranslateMemoryOperation(const IR::Instruction& inst);
bool TranslateIntegerCompare(const IR::Instruction& inst);
bool TranslateInteger16Compare(const IR::Instruction& inst);
bool TranslateFloatCompare(const IR::Instruction& inst);
bool TranslateConversion(const IR::Instruction& inst);
bool TranslateInteger16Operation(const IR::Instruction& inst);
bool TranslatePackedInteger16(const IR::Instruction& inst);
bool TranslatePackedFloat16(const IR::Instruction& inst);
bool TranslateFloat16Operation(const IR::Instruction& inst);
bool TranslateFloatOperation(const IR::Instruction& inst);
IR::U1 EvaluateU64Mask(const IR::Instruction& inst);
bool TranslateU64MaskOperation(const IR::Instruction& inst);
bool TranslateSimpleInteger(const IR::Instruction& inst);
bool TranslateComposedInteger(const IR::Instruction& inst);
bool TranslateExtendedInteger(const IR::Instruction& inst);
IR::ValueProgram& value_program;
IR::IREmitter ir;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
bool current_per_invocation_masks = false;
IR::Opcode current_opcode = IR::Opcode::ControlNop;
uint32_t current_pc = 0;
uint32_t current_vector_limit = 1;
uint32_t current_wave_size = 64;
};
} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail
src/graphics/shader/recompiler/ir/Block.h
+1
View File
@@ -49,6 +49,7 @@ public:
std::array<Value, NumScalarRegs> ssa_sreg_values {};
std::array<Value, NumScalarRegs> ssa_thread_bit_sreg_values {};
std::array<Value, NumScalarRegs> ssa_sreg_mask_tags {};
std::array<Value, NumVectorRegs> ssa_vreg_values {};
private:
src/graphics/shader/recompiler/ir/IREmitter.cpp
+8
View File
@@ -36,6 +36,14 @@ void IREmitter::SetThreadBitScalarReg(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetThreadBitScalarRegister, {Value(reg), value});
}
U1 IREmitter::GetScalarMaskTag(ScalarReg reg) {
return U1(Emit(ValueOpcode::GetScalarMaskTag, {Value(reg)}));
}
void IREmitter::SetScalarMaskTag(ScalarReg reg, U1 value) {
Emit(ValueOpcode::SetScalarMaskTag, {Value(reg), value});
}
U32 IREmitter::GetVectorReg(VectorReg reg) {
return U32(Emit(ValueOpcode::GetVectorRegister, {Value(reg)}));
}
src/graphics/shader/recompiler/ir/IREmitter.h
+2
View File
@@ -29,6 +29,8 @@ public:
void SetScalarReg(ScalarReg reg, U32 value);
U1 GetThreadBitScalarReg(ScalarReg reg);
void SetThreadBitScalarReg(ScalarReg reg, U1 value);
U1 GetScalarMaskTag(ScalarReg reg);
void SetScalarMaskTag(ScalarReg reg, U1 value);
U32 GetVectorReg(VectorReg reg);
void SetVectorReg(VectorReg reg, U32 value);
U1 GetGotoVariable(uint32_t id);
src/graphics/shader/recompiler/ir/ShaderIR.cpp
+28 -11
View File
@@ -44,6 +44,23 @@ bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBloc
if (!ScalarResultWritesSccNonZero(decoded.opcode)) {
return true;
}
switch (decoded.opcode) {
case Decoder::Opcode::S_AND_B64:
case Decoder::Opcode::S_ANDN2_B64:
case Decoder::Opcode::S_NOT_B64:
case Decoder::Opcode::S_OR_B64:
case Decoder::Opcode::S_ORN2_B64:
case Decoder::Opcode::S_XOR_B64:
case Decoder::Opcode::S_NAND_B64:
case Decoder::Opcode::S_NOR_B64:
case Decoder::Opcode::S_XNOR_B64:
case Decoder::Opcode::S_WQM_B64:
// These instructions maintain an invocation-local mask shadow and update SCC
// while lowering the operation. A raw dword comparison would disagree after
// complementing a mask temporary.
return true;
default: break;
}
if (block.instructions.empty()) {
SetError(error, "internal IR error: missing scalar result for SCC update");
return false;
@@ -206,14 +223,14 @@ uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand) {
MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind) {
MemoryInfo mem;
mem.kind = kind;
mem.offset = decoded.offset;
mem.secondary_offset = decoded.secondary_offset;
mem.dmask = decoded.dmask;
mem.data_dwords = decoded.data_dwords;
mem.data_bits = decoded.data_bits;
mem.component_count = decoded.data_components != 0u ? decoded.data_components
: decoded.data_dwords;
mem.kind = kind;
mem.offset = decoded.offset;
mem.secondary_offset = decoded.secondary_offset;
mem.dmask = decoded.dmask;
mem.data_dwords = decoded.data_dwords;
mem.data_bits = decoded.data_bits;
mem.component_count =
decoded.data_components != 0u ? decoded.data_components : decoded.data_dwords;
mem.data_format = decoded.data_format;
mem.number_format = decoded.number_format;
mem.image_sample_flags = decoded.image_sample_flags;
@@ -224,9 +241,9 @@ MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKi
mem.image_nsa_addr[i] = decoded.image_nsa_addr[i];
}
mem.memory_segment = decoded.memory_segment;
mem.address_is_full = kind == ResourceKind::Flat ||
(kind == ResourceKind::Global &&
decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.address_is_full =
kind == ResourceKind::Flat ||
(kind == ResourceKind::Global && decoded.src1.kind == Decoder::OperandKind::Vgpr);
mem.data_signed = decoded.data_signed;
mem.typed = decoded.typed;
mem.formatted = decoded.formatted;
src/graphics/shader/recompiler/ir/ShaderIR.h
-2
View File
@@ -456,7 +456,6 @@ struct ShaderInfo {
};
struct SpirvRequirements {
bool requires_exact_subgroup = false;
bool subgroup_ballot = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
@@ -469,7 +468,6 @@ struct SpirvRequirements {
struct Program {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
src/graphics/shader/recompiler/ir/ShaderIRLog.cpp
+1 -3
View File
@@ -192,10 +192,8 @@ std::string ExportTargetKindToString(ExportTargetKind kind) {
std::string ProgramToString(const Program& program) {
std::string text;
text += fmt::format("mode={} masks={} fallback={} reason={}\n",
text += fmt::format("mode={} fallback={} reason={}\n",
program.dispatcher_fallback ? "dispatcher" : "structured",
program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave"
: "per-invocation",
CFG::FailureKindToString(program.cfg_failure_kind).c_str(),
program.fallback_reason.c_str());
for (const auto& block: program.blocks) {
src/graphics/shader/recompiler/ir/ValueProgram.cpp
+2
View File
@@ -17,6 +17,8 @@ bool IsRegisterStatePseudo(ValueOpcode opcode) {
switch (opcode) {
case ValueOpcode::GetThreadBitScalarRegister:
case ValueOpcode::SetThreadBitScalarRegister:
case ValueOpcode::GetScalarMaskTag:
case ValueOpcode::SetScalarMaskTag:
case ValueOpcode::GetScalarRegister:
case ValueOpcode::SetScalarRegister:
case ValueOpcode::GetVectorRegister:
src/graphics/shader/recompiler/ir/opcodes/ValueOpcodes.inc
+2
View File
@@ -11,6 +11,8 @@ VALUE_OPCODE(GetShaderBase, U64)
VALUE_OPCODE(GetBuiltin, U32, U32, U32)
VALUE_OPCODE(GetThreadBitScalarRegister, U1, ScalarReg)
VALUE_OPCODE(SetThreadBitScalarRegister, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarMaskTag, U1, ScalarReg)
VALUE_OPCODE(SetScalarMaskTag, Void, ScalarReg, U1)
VALUE_OPCODE(GetScalarRegister, U32, ScalarReg)
VALUE_OPCODE(SetScalarRegister, Void, ScalarReg, U32)
VALUE_OPCODE(GetVectorRegister, U32, VectorReg)
src/graphics/shader/recompiler/ir/passes/SsaRewrite.cpp
+27 -2
View File
@@ -15,6 +15,10 @@ struct ThreadBitScalarReg {
ScalarReg reg {};
auto operator<=>(const ThreadBitScalarReg&) const = default;
};
struct ScalarMaskTag {
ScalarReg reg {};
auto operator<=>(const ScalarMaskTag&) const = default;
};
struct ExecTag {
auto operator<=>(const ExecTag&) const = default;
};
@@ -41,8 +45,9 @@ struct GotoVariable {
auto operator<=>(const GotoVariable&) const = default;
};
using Variable = std::variant<ScalarReg, ThreadBitScalarReg, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using Variable =
std::variant<ScalarReg, ThreadBitScalarReg, ScalarMaskTag, VectorReg, GotoVariable, SccTag,
ExecTag, ExecLoTag, ExecHiTag, VccTag, VccLoTag, VccHiTag, M0Tag>;
using ValueMap = std::unordered_map<Block*, Value>;
struct DefTable {
@@ -66,6 +71,14 @@ struct DefTable {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_thread_bit_sreg_values[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, ScalarMaskTag value) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
return block->ssa_sreg_mask_tags[RegIndex(value.reg)];
}
void Set(Block* block, ScalarMaskTag value, Value definition) {
EXIT_IF(RegIndex(value.reg) >= NumScalarRegs);
block->ssa_sreg_mask_tags[RegIndex(value.reg)] = definition;
}
const Value& Get(Block* block, VectorReg reg) {
EXIT_IF(RegIndex(reg) >= NumVectorRegs);
@@ -116,6 +129,9 @@ ValueOpcode UndefOpcode(ScalarReg) {
ValueOpcode UndefOpcode(ThreadBitScalarReg) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(ScalarMaskTag) {
return ValueOpcode::UndefU1;
}
ValueOpcode UndefOpcode(VectorReg) {
return ValueOpcode::UndefU32;
}
@@ -153,6 +169,9 @@ Value InitialValue(ScalarReg) {
Value InitialValue(ThreadBitScalarReg) {
return Value(false);
}
Value InitialValue(ScalarMaskTag) {
return Value(false);
}
Value InitialValue(VectorReg) {
return Value(0u);
}
@@ -321,6 +340,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
case ValueOpcode::SetThreadBitScalarRegister:
pass.Write(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetScalarMaskTag:
pass.Write(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block, inst.Arg(1));
break;
case ValueOpcode::SetVectorRegister:
pass.Write(inst.Arg(0).VectorRegister(), block, inst.Arg(1));
break;
@@ -342,6 +364,9 @@ void VisitInstruction(Pass& pass, Block* block, Inst& inst) {
inst.ReplaceUsesWith(
pass.Read(ThreadBitScalarReg {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetScalarMaskTag:
inst.ReplaceUsesWith(pass.Read(ScalarMaskTag {inst.Arg(0).ScalarRegister()}, block));
break;
case ValueOpcode::GetVectorRegister:
inst.ReplaceUsesWith(pass.Read(inst.Arg(0).VectorRegister(), block));
break;
src/graphics/shader/shader.cpp
+14 -28
View File
@@ -112,16 +112,14 @@ static std::unique_ptr<std::unordered_map<uint64_t, ShaderMappedData>> g_shader_
static std::mutex g_shader_map_mutex;
struct ShaderStageProgramKey {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
ShaderType stage = ShaderType::Unknown;
uint64_t shader_hash = 0;
ShaderId program_id;
Config::ShaderOptimizationType optimization_type = Config::ShaderOptimizationType::None;
bool operator==(const ShaderStageProgramKey& other) const {
return stage == other.stage && lane_mask_mode == other.lane_mask_mode &&
shader_hash == other.shader_hash && program_id == other.program_id &&
optimization_type == other.optimization_type;
return stage == other.stage && shader_hash == other.shader_hash &&
program_id == other.program_id && optimization_type == other.optimization_type;
}
};
@@ -133,7 +131,6 @@ struct ShaderStageProgramKeyHash {
};
mix(static_cast<uint32_t>(key.stage));
mix(static_cast<uint32_t>(key.lane_mask_mode));
mix(Common::hash64(key.shader_hash));
mix(key.program_id.hash0);
mix(key.program_id.crc32);
@@ -920,11 +917,9 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
}
static ShaderStageProgramKey MakeShaderStageProgramKey(ShaderType stage, uint64_t shader_hash,
const ShaderId& program_id,
ShaderLaneMaskMode lane_mask_mode) {
const ShaderId& program_id) {
ShaderStageProgramKey key {};
key.stage = stage;
key.lane_mask_mode = lane_mask_mode;
key.shader_hash = shader_hash;
key.program_id = program_id;
key.optimization_type = Config::GetShaderOptimizationType();
@@ -1055,7 +1050,6 @@ static std::string ShaderDescribeSpecialization(const ShaderRecompiler::IR::Prog
static void ShaderAppendNativeSpecialization(std::vector<uint32_t>& ids,
const ShaderRecompiler::IR::Program& program) {
EXIT_IF(!program.binding_layout_complete);
ids.push_back(static_cast<uint32_t>(program.lane_mask_mode));
ids.push_back(program.bindings.descriptor_set);
ids.push_back(program.bindings.push_constant_offset);
ids.push_back(program.bindings.push_constant_size);
@@ -1136,8 +1130,7 @@ static std::span<const uint32_t> AddShaderProgramPermutation(const char* stage,
}
bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& info,
std::span<const uint32_t>& spirv) {
ShaderVertexInputInfo& info, std::span<const uint32_t>& spirv) {
spirv = {};
if (!ShaderGetStaticInputInfoVS(regs, sh, info)) {
@@ -1145,8 +1138,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis
}
const auto shader_hash = regs.gs_regs.chksum;
const auto program_id = ShaderGetIdVS(regs, info, false);
const auto key =
MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id, lane_mask_mode);
const auto key = MakeShaderStageProgramKey(ShaderType::Vertex, shader_hash, program_id);
{
std::scoped_lock lock(g_shader_program_cache_mutex);
@@ -1163,7 +1155,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis
}
std::vector<uint32_t> compiled_spirv;
if (!ShaderCompileSpirvVS(regs, sh, lane_mask_mode, info, compiled_spirv)) {
if (!ShaderCompileSpirvVS(regs, sh, info, compiled_spirv)) {
return false;
}
@@ -1175,7 +1167,7 @@ bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegis
}
bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info,
const ShaderVertexInputInfo& vs_info,
std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
ShaderPixelInputInfo& ps_info, std::span<const uint32_t>& spirv) {
spirv = {};
@@ -1184,8 +1176,7 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist
const auto shader_hash =
regs.ps_regs.chksum != 0 ? regs.ps_regs.chksum : regs.ps_regs.data_addr;
const auto program_id = ShaderGetIdPS(regs, ps_info, false);
const auto key =
MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id, lane_mask_mode);
const auto key = MakeShaderStageProgramKey(ShaderType::Pixel, shader_hash, program_id);
{
std::scoped_lock lock(g_shader_program_cache_mutex);
@@ -1202,7 +1193,7 @@ bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegist
}
std::vector<uint32_t> compiled_spirv;
if (!ShaderCompileSpirvPS(regs, sh, lane_mask_mode, ps_info, compiled_spirv)) {
if (!ShaderCompileSpirvPS(regs, sh, ps_info, compiled_spirv)) {
return false;
}
@@ -1220,8 +1211,7 @@ bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegi
ShaderGetStaticInputInfoCS(regs, sh, info);
const auto shader_hash = regs.cs_regs.data_addr;
const auto program_id = ShaderGetIdCS(regs, info, false);
const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id,
ShaderLaneMaskMode::NativeWave);
const auto key = MakeShaderStageProgramKey(ShaderType::Compute, shader_hash, program_id);
{
std::scoped_lock lock(g_shader_program_cache_mutex);
@@ -1449,8 +1439,7 @@ static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash,
}
bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::vector<uint32_t>& spirv) {
ShaderVertexInputInfo& input_info, std::vector<uint32_t>& spirv) {
KYTY_PROFILER_FUNCTION(profiler::colors::Amber300);
EXIT_NOT_IMPLEMENTED(regs.es_regs.data_addr == 0 || regs.gs_regs.chksum == 0);
@@ -1460,7 +1449,6 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Vertex;
options.lane_mask_mode = lane_mask_mode;
options.shader_hash = regs.gs_regs.chksum;
options.shader_base = shader_addr;
options.user_data_base = 8;
@@ -1506,8 +1494,7 @@ bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegi
}
bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info,
std::vector<uint32_t>& spirv) {
ShaderPixelInputInfo& input_info, std::vector<uint32_t>& spirv) {
KYTY_PROFILER_FUNCTION(profiler::colors::Blue300);
const uint64_t shader_addr = regs.ps_regs.data_addr;
@@ -1516,7 +1503,6 @@ bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegis
ShaderRecompiler::CompileOptions options;
options.stage = ShaderType::Pixel;
options.lane_mask_mode = lane_mask_mode;
options.shader_hash = shader_hash;
options.shader_base = shader_addr;
options.user_data_count = regs.ps_regs.rsrc2.user_sgpr;
src/graphics/shader/shader.h
+4 -9
View File
@@ -35,8 +35,6 @@ struct ShaderRegisters;
enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute };
enum class ShaderLaneMaskMode { NativeWave, PerInvocation };
namespace ShaderRecompiler::IR {
struct Program;
struct ResourceSnapshot;
@@ -261,20 +259,17 @@ ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInp
bool include_bind_specialization);
// Returned SPIR-V spans are read-only views backed by the shader program cache.
bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::span<const uint32_t>& spirv);
ShaderVertexInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info,
const ShaderVertexInputInfo& vs_info,
std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
ShaderPixelInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderComputeInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::vector<uint32_t>& spirv);
ShaderVertexInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info,
std::vector<uint32_t>& spirv);
ShaderPixelInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderComputeInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderAddressValid(uint64_t addr);