mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-29 14:26:53 +01:00
shader_recompiler: add SSA
This commit is contained in:
nmzik
@@ -0,0 +1,160 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
static constexpr size_t InitialSpirvSectionReserve = 4096;
|
||||
static constexpr size_t InitialSpirvFunctionSectionReserve = 450000;
|
||||
|
||||
static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_t* words,
|
||||
size_t words_num) {
|
||||
if (words_num == 0) {
|
||||
return;
|
||||
}
|
||||
const auto opcode = words[0];
|
||||
const auto word_count = static_cast<uint32_t>(words_num);
|
||||
section.push_back((word_count << 16u) | opcode);
|
||||
section.insert(section.end(), words + 1, words + words_num);
|
||||
}
|
||||
|
||||
Builder::Builder(uint32_t version): m_version(version) {
|
||||
m_debug.reserve(InitialSpirvSectionReserve);
|
||||
m_annotations.reserve(InitialSpirvSectionReserve);
|
||||
m_types.reserve(InitialSpirvSectionReserve);
|
||||
m_functions.reserve(InitialSpirvFunctionSectionReserve);
|
||||
}
|
||||
|
||||
uint32_t Builder::AllocateId() {
|
||||
return m_next_id++;
|
||||
}
|
||||
|
||||
void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
|
||||
const auto len = text != nullptr ? std::strlen(text) : 0;
|
||||
const auto word_count = (len + 1u + 3u) / 4u;
|
||||
for (size_t i = 0; i < word_count; i++) {
|
||||
uint32_t word = 0;
|
||||
for (size_t byte = 0; byte < 4; byte++) {
|
||||
const auto index = i * 4u + byte;
|
||||
if (index < len) {
|
||||
word |= static_cast<uint32_t>(static_cast<unsigned char>(text[index]))
|
||||
<< (byte * 8u);
|
||||
}
|
||||
}
|
||||
words.push_back(word);
|
||||
}
|
||||
}
|
||||
|
||||
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
const std::vector<uint32_t>& operands) {
|
||||
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
|
||||
section.push_back((word_count << 16u) | opcode);
|
||||
section.insert(section.end(), operands.begin(), operands.end());
|
||||
}
|
||||
|
||||
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
std::initializer_list<uint32_t> operands) {
|
||||
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
|
||||
section.push_back((word_count << 16u) | opcode);
|
||||
section.insert(section.end(), operands.begin(), operands.end());
|
||||
}
|
||||
|
||||
void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
|
||||
AppendInstruction(m_capabilities, 17u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddExtension(const char* name) {
|
||||
std::vector<uint32_t> operands;
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_extensions, 10u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddExtInstImport(uint32_t id, const char* name) {
|
||||
std::vector<uint32_t> operands = {id};
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_ext_inst_imports, 11u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
|
||||
AppendInstruction(m_memory_model, 14u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
|
||||
const std::vector<uint32_t>& interfaces) {
|
||||
std::vector<uint32_t> operands = {execution_model, entry_point};
|
||||
AppendString(operands, name);
|
||||
operands.insert(operands.end(), interfaces.begin(), interfaces.end());
|
||||
AppendInstruction(m_entry_points, 15u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
|
||||
AppendInstruction(m_execution_modes, 16u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddName(uint32_t target, const char* name) {
|
||||
std::vector<uint32_t> operands = {target};
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_debug, 5u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
|
||||
AppendInstructionWords(m_annotations, words.begin(), words.size());
|
||||
}
|
||||
|
||||
void Builder::AddType(std::initializer_list<uint32_t> words) {
|
||||
AppendInstructionWords(m_types, words.begin(), words.size());
|
||||
}
|
||||
|
||||
void Builder::AddFunction(std::initializer_list<uint32_t> words) {
|
||||
AppendInstructionWords(m_functions, words.begin(), words.size());
|
||||
}
|
||||
|
||||
void Builder::AddFunction(const std::vector<uint32_t>& words) {
|
||||
AppendInstructionWords(m_functions, words.data(), words.size());
|
||||
}
|
||||
|
||||
std::vector<uint32_t> Builder::Build() const {
|
||||
if (m_debug.size() > InitialSpirvSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=debug size=%zu reserve=%zu\n",
|
||||
m_debug.size(), InitialSpirvSectionReserve);
|
||||
}
|
||||
if (m_annotations.size() > InitialSpirvSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=annotations size=%zu reserve=%zu\n",
|
||||
m_annotations.size(), InitialSpirvSectionReserve);
|
||||
}
|
||||
if (m_types.size() > InitialSpirvSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=types size=%zu reserve=%zu\n",
|
||||
m_types.size(), InitialSpirvSectionReserve);
|
||||
}
|
||||
if (m_functions.size() > InitialSpirvFunctionSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=functions size=%zu reserve=%zu\n",
|
||||
m_functions.size(), InitialSpirvFunctionSectionReserve);
|
||||
}
|
||||
|
||||
std::vector<uint32_t> module;
|
||||
module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() +
|
||||
m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() +
|
||||
m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size());
|
||||
|
||||
module.push_back(0x07230203u);
|
||||
module.push_back(m_version);
|
||||
module.push_back(0u);
|
||||
module.push_back(m_next_id);
|
||||
module.push_back(0u);
|
||||
|
||||
module.insert(module.end(), m_capabilities.begin(), m_capabilities.end());
|
||||
module.insert(module.end(), m_extensions.begin(), m_extensions.end());
|
||||
module.insert(module.end(), m_ext_inst_imports.begin(), m_ext_inst_imports.end());
|
||||
module.insert(module.end(), m_memory_model.begin(), m_memory_model.end());
|
||||
module.insert(module.end(), m_entry_points.begin(), m_entry_points.end());
|
||||
module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end());
|
||||
module.insert(module.end(), m_debug.begin(), m_debug.end());
|
||||
module.insert(module.end(), m_annotations.begin(), m_annotations.end());
|
||||
module.insert(module.end(), m_types.begin(), m_types.end());
|
||||
module.insert(module.end(), m_functions.begin(), m_functions.end());
|
||||
|
||||
return module;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
@@ -0,0 +1,57 @@
|
||||
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
|
||||
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
|
||||
|
||||
#include "common/common.h"
|
||||
|
||||
#include <initializer_list>
|
||||
#include <vector>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
class Builder {
|
||||
public:
|
||||
explicit Builder(uint32_t version = 0x00010300u);
|
||||
~Builder() = default;
|
||||
KYTY_CLASS_DEFAULT_COPY(Builder);
|
||||
|
||||
uint32_t AllocateId();
|
||||
|
||||
void AddCapability(std::initializer_list<uint32_t> operands);
|
||||
void AddExtension(const char* name);
|
||||
void AddExtInstImport(uint32_t id, const char* name);
|
||||
void AddMemoryModel(std::initializer_list<uint32_t> operands);
|
||||
void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
|
||||
const std::vector<uint32_t>& interfaces);
|
||||
void AddExecutionMode(std::initializer_list<uint32_t> operands);
|
||||
void AddName(uint32_t target, const char* name);
|
||||
void AddAnnotation(std::initializer_list<uint32_t> words);
|
||||
void AddType(std::initializer_list<uint32_t> words);
|
||||
void AddFunction(std::initializer_list<uint32_t> words);
|
||||
void AddFunction(const std::vector<uint32_t>& words);
|
||||
|
||||
[[nodiscard]] std::vector<uint32_t> Build() const;
|
||||
|
||||
private:
|
||||
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
const std::vector<uint32_t>& operands);
|
||||
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
std::initializer_list<uint32_t> operands);
|
||||
static void AppendString(std::vector<uint32_t>& words, const char* text);
|
||||
|
||||
uint32_t m_next_id = 1;
|
||||
uint32_t m_version = 0;
|
||||
std::vector<uint32_t> m_capabilities;
|
||||
std::vector<uint32_t> m_extensions;
|
||||
std::vector<uint32_t> m_ext_inst_imports;
|
||||
std::vector<uint32_t> m_memory_model;
|
||||
std::vector<uint32_t> m_entry_points;
|
||||
std::vector<uint32_t> m_execution_modes;
|
||||
std::vector<uint32_t> m_debug;
|
||||
std::vector<uint32_t> m_annotations;
|
||||
std::vector<uint32_t> m_types;
|
||||
std::vector<uint32_t> m_functions;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ */
|
||||
@@ -0,0 +1,396 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
|
||||
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
#include "graphics/shader/recompiler/ir/ValueProgram.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cinttypes>
|
||||
#include <cstdio>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
namespace {
|
||||
|
||||
bool Fail(std::string* error, const std::string& message) {
|
||||
if (error != nullptr) {
|
||||
*error = message;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
|
||||
using Kind = IR::DescriptorBindingKind;
|
||||
using Dim = Decoder::ImageDimension;
|
||||
if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
|
||||
const bool integer = image.kind == IR::ResourceKind::ImageUint;
|
||||
switch (image.dimension) {
|
||||
case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
|
||||
case Dim::Dim1DArray:
|
||||
kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
|
||||
return true;
|
||||
case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
|
||||
case Dim::Dim2DMsaa:
|
||||
kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
|
||||
return true;
|
||||
case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
|
||||
case Dim::Dim2DArray:
|
||||
kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
|
||||
return true;
|
||||
case Dim::Dim2DMsaaArray:
|
||||
kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
|
||||
return true;
|
||||
case Dim::Unknown: return false;
|
||||
}
|
||||
}
|
||||
const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
|
||||
if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
|
||||
return false;
|
||||
}
|
||||
switch (image.dimension) {
|
||||
case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
|
||||
case Dim::Dim1DArray:
|
||||
kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
|
||||
return true;
|
||||
case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
|
||||
case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
|
||||
case Dim::Dim2DArray:
|
||||
kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
|
||||
return true;
|
||||
case Dim::Dim2DMsaa:
|
||||
case Dim::Dim2DMsaaArray: return false;
|
||||
case Dim::Unknown: return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
|
||||
using Kind = IR::DescriptorBindingKind;
|
||||
constexpr auto KindCount = static_cast<size_t>(Kind::Count);
|
||||
std::array<std::vector<uint32_t>, KindCount> expected;
|
||||
std::array<bool, KindCount> present {};
|
||||
const auto Dense = [](size_t size) {
|
||||
std::vector<uint32_t> values(size);
|
||||
for (uint32_t i = 0; i < values.size(); i++) {
|
||||
values[i] = i;
|
||||
}
|
||||
return values;
|
||||
};
|
||||
auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
|
||||
const auto index = static_cast<size_t>(kind);
|
||||
present[index] = true;
|
||||
expected[index] = std::move(resources);
|
||||
};
|
||||
if (!program.info.buffers.empty()) {
|
||||
Expect(Kind::Buffers, Dense(program.info.buffers.size()));
|
||||
}
|
||||
for (uint32_t i = 0; i < program.info.images.size(); i++) {
|
||||
Kind kind;
|
||||
if (!ImageBinding(program.info.images[i], kind)) {
|
||||
return Fail(error, "native shader plan has an invalid image class");
|
||||
}
|
||||
present[static_cast<size_t>(kind)] = true;
|
||||
expected[static_cast<size_t>(kind)].push_back(i);
|
||||
}
|
||||
if (!program.info.samplers.empty()) {
|
||||
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
|
||||
}
|
||||
const auto uses_gds = program.values != nullptr &&
|
||||
std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
|
||||
return std::ranges::any_of(*block, [](const IR::Inst& inst) {
|
||||
return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
|
||||
});
|
||||
});
|
||||
if (uses_gds) {
|
||||
Expect(Kind::Gds);
|
||||
}
|
||||
if (!program.info.addresses.empty()) {
|
||||
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
|
||||
}
|
||||
if (program.values != nullptr && !program.values->srt_reads.empty()) {
|
||||
Expect(Kind::FlattenedSrt);
|
||||
}
|
||||
if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
|
||||
Expect(Kind::UserData);
|
||||
}
|
||||
|
||||
std::array<bool, KindCount> seen {};
|
||||
for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
|
||||
const auto& binding = program.bindings.descriptors[i];
|
||||
const auto kind = static_cast<size_t>(binding.kind);
|
||||
if (kind >= KindCount || seen[kind] || !present[kind] ||
|
||||
binding.resources != expected[kind]) {
|
||||
return Fail(error, "native descriptor groups do not match shader topology");
|
||||
}
|
||||
for (uint32_t j = 0; j < i; j++) {
|
||||
if (program.bindings.descriptors[j].binding == binding.binding) {
|
||||
return Fail(error, "native descriptor binding numbers are not unique");
|
||||
}
|
||||
}
|
||||
seen[kind] = true;
|
||||
}
|
||||
for (size_t i = 0; i < KindCount; i++) {
|
||||
if (present[i] != seen[i]) {
|
||||
return Fail(error, "native shader plan is missing a required descriptor group");
|
||||
}
|
||||
}
|
||||
const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
|
||||
if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
|
||||
program.bindings.buffer_offset_count != program.info.buffers.size() ||
|
||||
(!has_user_storage &&
|
||||
program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
|
||||
(has_user_storage && program.bindings.push_constant_size != 0) ||
|
||||
!std::is_sorted(program.bindings.user_data_registers.begin(),
|
||||
program.bindings.user_data_registers.end()) ||
|
||||
std::adjacent_find(program.bindings.user_data_registers.begin(),
|
||||
program.bindings.user_data_registers.end()) !=
|
||||
program.bindings.user_data_registers.end()) {
|
||||
return Fail(error, "native user-data layout is inconsistent");
|
||||
}
|
||||
|
||||
if (program.values == nullptr) {
|
||||
return Fail(error, "native shader plan has no typed SSA");
|
||||
}
|
||||
const auto planning_only_handle = [&](const IR::Inst& handle) {
|
||||
return !handle.Uses().empty() &&
|
||||
std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
|
||||
const auto op = use.user->GetOpcode();
|
||||
if (op != IR::ValueOpcode::LoadAddressU32 &&
|
||||
op != IR::ValueOpcode::ReadConstBuffer) {
|
||||
return false;
|
||||
}
|
||||
const auto index = use.user->Flags<IR::MemoryFlags>().index;
|
||||
return index < program.values->memory_info.size() &&
|
||||
program.values->memory_info[index].planning_only;
|
||||
});
|
||||
};
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
const auto dense = inst.Flags<uint32_t>();
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::GetBufferResource:
|
||||
if (planning_only_handle(inst)) {
|
||||
break;
|
||||
}
|
||||
if (dense >= program.info.buffers.size()) {
|
||||
return Fail(error, "typed buffer handle has an invalid dense resource");
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::GetAddressResource:
|
||||
if (planning_only_handle(inst)) {
|
||||
break;
|
||||
}
|
||||
if (dense >= program.info.addresses.size()) {
|
||||
return Fail(error, "typed address handle has an invalid dense resource");
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::GetImageResource:
|
||||
if (dense >= program.info.images.size()) {
|
||||
return Fail(error, "typed image handle has an invalid dense resource");
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::GetSamplerResource:
|
||||
if (dense >= program.info.samplers.size()) {
|
||||
return Fail(error, "typed sampler handle has an invalid dense resource");
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::ReadConst: {
|
||||
const auto slot = inst.Arg(1).Resolve();
|
||||
if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
|
||||
slot.U32() >= program.values->srt_reads.size()) {
|
||||
return Fail(error, "flattened SRT read has an invalid dense slot");
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
static constexpr size_t InitialEmitterVectorReserve = 4096;
|
||||
|
||||
static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
|
||||
size_t size) {
|
||||
if (size <= InitialEmitterVectorReserve) {
|
||||
return;
|
||||
}
|
||||
std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
|
||||
" stage=%u vector=%s size=%zu reserve=%zu\n",
|
||||
program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
|
||||
InitialEmitterVectorReserve);
|
||||
}
|
||||
|
||||
void CollectValueRegisterMirrors(const IR::ValueProgram& program,
|
||||
std::vector<Emitter::RegisterBinding>& registers) {
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::SetScalarRegister:
|
||||
case IR::ValueOpcode::SetThreadBitScalarRegister:
|
||||
if (block != program.blocks.front()) {
|
||||
Emitter::CollectRegister(
|
||||
registers,
|
||||
{IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())});
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::SetVectorRegister:
|
||||
Emitter::CollectRegister(
|
||||
registers,
|
||||
{IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())});
|
||||
break;
|
||||
case IR::ValueOpcode::SetM0:
|
||||
if (block != program.blocks.front()) {
|
||||
Emitter::CollectRegister(registers, {IR::RegisterFile::M0, 0});
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::SetExecLo:
|
||||
case IR::ValueOpcode::SetExecHi:
|
||||
if (block != program.blocks.front()) {
|
||||
Emitter::CollectRegister(
|
||||
registers, {IR::RegisterFile::Exec,
|
||||
inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u});
|
||||
}
|
||||
break;
|
||||
case IR::ValueOpcode::SetVccLo:
|
||||
case IR::ValueOpcode::SetVccHi:
|
||||
if (block != program.blocks.front()) {
|
||||
Emitter::CollectRegister(
|
||||
registers, {IR::RegisterFile::Vcc,
|
||||
inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u});
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool ValueProgramHasOpcode(const IR::ValueProgram& program, IR::ValueOpcode opcode) {
|
||||
return std::ranges::any_of(program.blocks, [opcode](const IR::Block* block) {
|
||||
return std::ranges::any_of(
|
||||
*block, [opcode](const IR::Inst& inst) { return inst.GetOpcode() == opcode; });
|
||||
});
|
||||
}
|
||||
|
||||
bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
|
||||
if (program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::Ballot:
|
||||
case IR::ValueOpcode::DppMoveU32:
|
||||
case IR::ValueOpcode::DppUpdateU32:
|
||||
case IR::ValueOpcode::WqmMask:
|
||||
case IR::ValueOpcode::ReadFirstLane:
|
||||
case IR::ValueOpcode::ReadLane:
|
||||
case IR::ValueOpcode::WriteLane:
|
||||
case IR::ValueOpcode::Permlane16U32:
|
||||
case IR::ValueOpcode::SwizzleU32:
|
||||
case IR::ValueOpcode::DataAppend:
|
||||
case IR::ValueOpcode::DataConsume:
|
||||
case IR::ValueOpcode::GdsDataAppend:
|
||||
case IR::ValueOpcode::GdsDataConsume: return true;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
|
||||
const ShaderVertexInputInfo* vertex_input_info,
|
||||
const ShaderPixelInputInfo* pixel_input_info,
|
||||
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
|
||||
std::string* error, bool preserve_debug_values) {
|
||||
using namespace Emitter;
|
||||
|
||||
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
|
||||
program.stage != ShaderType::Pixel) {
|
||||
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
|
||||
return false;
|
||||
}
|
||||
if (!program.srt_plan_complete || !program.resource_tracking_complete ||
|
||||
!program.shader_info_complete || !program.binding_layout_complete) {
|
||||
SetError(error, "SPIR-V emitter requires a fully planned native shader program");
|
||||
return false;
|
||||
}
|
||||
if (!IR::ValidateResourceSnapshot(program, resources, error)) {
|
||||
return false;
|
||||
}
|
||||
if (!IR::ValidateResourceSpecialization(program, resources, error)) {
|
||||
return false;
|
||||
}
|
||||
if (!ValidateNativeProgram(program, error)) {
|
||||
return false;
|
||||
}
|
||||
if (program.values == nullptr) {
|
||||
SetError(error, "SPIR-V emitter requires planned typed SSA");
|
||||
return false;
|
||||
}
|
||||
ShaderVertexInputInfo default_vertex {};
|
||||
ShaderPixelInputInfo default_pixel {};
|
||||
ShaderComputeInputInfo default_compute {};
|
||||
const auto* vertex = vertex_input_info != nullptr ? vertex_input_info : &default_vertex;
|
||||
const auto* pixel = pixel_input_info != nullptr ? pixel_input_info : &default_pixel;
|
||||
const auto* compute = compute_input_info != nullptr ? compute_input_info : &default_compute;
|
||||
(void)preserve_debug_values;
|
||||
const auto& value_program = *program.values;
|
||||
if (!IR::ValidateValueProgram(value_program, true, error)) {
|
||||
return false;
|
||||
}
|
||||
EmitterState state(program, resources);
|
||||
state.vertex_input_info = vertex;
|
||||
state.pixel_input_info = pixel;
|
||||
state.compute_input_info = compute;
|
||||
state.stage = program.stage;
|
||||
state.wave_size = program.wave_size;
|
||||
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
|
||||
state.exact_subgroup_operations =
|
||||
!state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
|
||||
state.registers.reserve(InitialEmitterVectorReserve);
|
||||
state.inputs.reserve(InitialEmitterVectorReserve);
|
||||
state.outputs.reserve(InitialEmitterVectorReserve);
|
||||
state.interface_variables.reserve(InitialEmitterVectorReserve);
|
||||
state.reachable_blocks.reserve(InitialEmitterVectorReserve);
|
||||
CollectValueRegisterMirrors(value_program, state.registers);
|
||||
CopyProgramInputsAndOutputs(state, program);
|
||||
state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program) ||
|
||||
(!state.per_invocation_masks &&
|
||||
ValueProgramHasOpcode(value_program, IR::ValueOpcode::Ballot));
|
||||
state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
|
||||
state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
|
||||
state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program);
|
||||
state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program);
|
||||
state.needs_function_lds = ProgramNeedsFunctionLds(program);
|
||||
state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program);
|
||||
AllocateInputVariables(state);
|
||||
AllocateOutputVariables(state);
|
||||
AllocateDescriptorVariables(state);
|
||||
EmitHeaderAndTypes(state);
|
||||
AllocateRegisterVariables(state);
|
||||
if (!EmitValueProgram(state, value_program, error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ReportReserveExceeded(program, "registers", state.registers.size());
|
||||
ReportReserveExceeded(program, "inputs", state.inputs.size());
|
||||
ReportReserveExceeded(program, "outputs", state.outputs.size());
|
||||
ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
|
||||
ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
|
||||
|
||||
auto binary = state.builder.Build();
|
||||
if (binary.empty()) {
|
||||
SetError(error, "SPIR-V builder returned an empty module");
|
||||
return false;
|
||||
}
|
||||
spirv = std::move(binary);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
@@ -0,0 +1,22 @@
|
||||
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_
|
||||
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_
|
||||
|
||||
#include "common/common.h"
|
||||
#include "common/stringUtils.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
|
||||
|
||||
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
|
||||
const ShaderVertexInputInfo* vertex_input_info,
|
||||
const ShaderPixelInputInfo* pixel_input_info,
|
||||
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
|
||||
std::string* error, bool preserve_debug_values = false);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_ */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,480 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
#include "graphics/shader/recompiler/ir/ValueProgram.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
|
||||
const auto* ps = state.pixel_input_info;
|
||||
if (state.stage != ShaderType::Pixel || ps == nullptr) {
|
||||
return attr;
|
||||
}
|
||||
return ShaderPixelParameterMappedLocation(*ps, attr);
|
||||
}
|
||||
|
||||
uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
|
||||
std::array<uint32_t, 32> active_inputs {};
|
||||
uint32_t active_count = 0;
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter) {
|
||||
active_inputs[active_count++] = input.location;
|
||||
}
|
||||
}
|
||||
return state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr
|
||||
? ShaderPixelParameterLocation(*state.pixel_input_info,
|
||||
{active_inputs.data(), active_count}, attr)
|
||||
: attr;
|
||||
}
|
||||
|
||||
bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
|
||||
const auto* ps = state.pixel_input_info;
|
||||
return state.stage == ShaderType::Pixel && ps != nullptr &&
|
||||
ShaderPixelParameterIsFlat(*ps, attr);
|
||||
}
|
||||
|
||||
void SetError(std::string* error, const char* message) {
|
||||
if (error != nullptr) {
|
||||
*error = message;
|
||||
}
|
||||
}
|
||||
|
||||
void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg) {
|
||||
if (std::any_of(registers.begin(), registers.end(),
|
||||
[reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
|
||||
return;
|
||||
}
|
||||
registers.push_back({reg, 0});
|
||||
}
|
||||
|
||||
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
|
||||
IR::Operand operand;
|
||||
operand.kind = IR::OperandKind::Register;
|
||||
operand.reg = {file, index};
|
||||
return operand;
|
||||
}
|
||||
|
||||
IR::Register SccRegister() {
|
||||
return {IR::RegisterFile::Scc, static_cast<uint32_t>(Decoder::OperandKind::Scc)};
|
||||
}
|
||||
|
||||
IR::Operand SccOperand() {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Scc,
|
||||
static_cast<uint32_t>(Decoder::OperandKind::Scc));
|
||||
}
|
||||
|
||||
bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
|
||||
return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
|
||||
}
|
||||
|
||||
bool IsMaskRegisterFile(IR::RegisterFile file) {
|
||||
return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc;
|
||||
}
|
||||
|
||||
bool IsSccOperand(const IR::Operand& operand) {
|
||||
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
|
||||
}
|
||||
|
||||
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
|
||||
uint32_t index) {
|
||||
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
|
||||
return binding.kind == kind && binding.index == index;
|
||||
});
|
||||
}
|
||||
|
||||
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
|
||||
for (const auto& input: program.info.inputs) {
|
||||
state.inputs.push_back(
|
||||
{input.kind, input.location, input.component_count, 0, input.debug_name});
|
||||
}
|
||||
for (const auto& output: program.info.outputs) {
|
||||
if (HasOutput(state.outputs, output.kind, output.index)) {
|
||||
continue;
|
||||
}
|
||||
state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name});
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
|
||||
if (exp.kind == IR::ExportTargetKind::Position) {
|
||||
return state.per_vertex_variable;
|
||||
}
|
||||
if (exp.kind == IR::ExportTargetKind::MrtZ) {
|
||||
return state.depth_variable;
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
const auto expected_kind = exp.kind == IR::ExportTargetKind::Mrt
|
||||
? IR::StageOutputKind::Mrt
|
||||
: IR::StageOutputKind::Parameter;
|
||||
if (binding.kind == expected_kind && binding.index == exp.index) {
|
||||
return binding.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
|
||||
if (program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (inst.GetOpcode() == IR::ValueOpcode::ImageQueryLod) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
|
||||
if (program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (inst.GetOpcode() == IR::ValueOpcode::ImageGatherRaw) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsFunctionLds(const IR::Program& program) {
|
||||
if (program.stage == ShaderType::Compute || program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::SwizzleU32: return true;
|
||||
case IR::ValueOpcode::LoadSharedU8:
|
||||
case IR::ValueOpcode::LoadSharedU16:
|
||||
case IR::ValueOpcode::LoadSharedU32:
|
||||
case IR::ValueOpcode::WriteSharedU8:
|
||||
case IR::ValueOpcode::WriteSharedU16:
|
||||
case IR::ValueOpcode::WriteSharedU32:
|
||||
case IR::ValueOpcode::SharedAtomicFMin32:
|
||||
case IR::ValueOpcode::SharedAtomicFMax32:
|
||||
case IR::ValueOpcode::SharedAtomicSwap32:
|
||||
case IR::ValueOpcode::SharedAtomicIAdd32:
|
||||
case IR::ValueOpcode::SharedAtomicISub32:
|
||||
case IR::ValueOpcode::SharedAtomicSMin32:
|
||||
case IR::ValueOpcode::SharedAtomicUMin32:
|
||||
case IR::ValueOpcode::SharedAtomicSMax32:
|
||||
case IR::ValueOpcode::SharedAtomicUMax32:
|
||||
case IR::ValueOpcode::SharedAtomicAnd32:
|
||||
case IR::ValueOpcode::SharedAtomicOr32:
|
||||
case IR::ValueOpcode::SharedAtomicXor32:
|
||||
case IR::ValueOpcode::DataAppend:
|
||||
case IR::ValueOpcode::DataConsume: {
|
||||
const auto index = inst.Flags<IR::MemoryFlags>().index;
|
||||
if (index < program.values->memory_info.size() &&
|
||||
program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsPixelValidMask(const IR::Program& program) {
|
||||
if (program.stage != ShaderType::Pixel || program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (inst.GetOpcode() != IR::ValueOpcode::SetAttribute) {
|
||||
continue;
|
||||
}
|
||||
const auto index = inst.Flags<IR::ExportFlags>().index;
|
||||
if (index < program.values->export_info.size() &&
|
||||
program.values->export_info[index].vm) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
|
||||
if (program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (inst.GetOpcode() == IR::ValueOpcode::WqmMask) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ProgramRequiresExactSubgroupSize(program);
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
|
||||
if (program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::DppMoveU32:
|
||||
case IR::ValueOpcode::ReadFirstLane:
|
||||
case IR::ValueOpcode::ReadLane:
|
||||
case IR::ValueOpcode::Permlane16U32:
|
||||
case IR::ValueOpcode::DataAppend:
|
||||
case IR::ValueOpcode::DataConsume:
|
||||
case IR::ValueOpcode::GdsDataAppend:
|
||||
case IR::ValueOpcode::GdsDataConsume:
|
||||
case IR::ValueOpcode::SwizzleU32: return true;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
|
||||
if (program.values == nullptr) {
|
||||
return false;
|
||||
}
|
||||
for (const auto* block: program.values->blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::DppMoveU32:
|
||||
case IR::ValueOpcode::DppUpdateU32:
|
||||
case IR::ValueOpcode::WqmMask:
|
||||
case IR::ValueOpcode::LaneId:
|
||||
case IR::ValueOpcode::WriteLane:
|
||||
case IR::ValueOpcode::Permlane16U32:
|
||||
case IR::ValueOpcode::DataAppend:
|
||||
case IR::ValueOpcode::DataConsume:
|
||||
case IR::ValueOpcode::GdsDataAppend:
|
||||
case IR::ValueOpcode::GdsDataConsume:
|
||||
case IR::ValueOpcode::SwizzleU32: return true;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg == reg) {
|
||||
return binding.pointer_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState& state, uint32_t value);
|
||||
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
|
||||
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* reason) {
|
||||
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
|
||||
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
|
||||
state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
|
||||
static_cast<unsigned>(kind), reason);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource) {
|
||||
const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
|
||||
if (descriptor == nullptr) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
|
||||
}
|
||||
const auto found =
|
||||
std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
|
||||
if (found == descriptor->resources.end()) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource,
|
||||
"resource is absent from descriptor group");
|
||||
}
|
||||
return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
|
||||
}
|
||||
|
||||
uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
|
||||
uint32_t variable_id, uint32_t array_index,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* variable_name) {
|
||||
if (variable_id == 0) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource, variable_name);
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
ImageViewKind ImageViewKindFromDimension(Decoder::ImageDimension dimension) {
|
||||
switch (dimension) {
|
||||
case Decoder::ImageDimension::Dim1D: return ImageViewKind::Dim1D;
|
||||
case Decoder::ImageDimension::Dim1DArray: return ImageViewKind::Dim1DArray;
|
||||
case Decoder::ImageDimension::Dim2DArray: return ImageViewKind::Dim2DArray;
|
||||
case Decoder::ImageDimension::Dim3D: return ImageViewKind::Dim3D;
|
||||
case Decoder::ImageDimension::Dim2DMsaa: return ImageViewKind::Dim2DMsaa;
|
||||
case Decoder::ImageDimension::Dim2DMsaaArray: return ImageViewKind::Dim2DMsaaArray;
|
||||
default: return ImageViewKind::Dim2D;
|
||||
}
|
||||
}
|
||||
|
||||
ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc) {
|
||||
(void)state;
|
||||
(void)use_pc;
|
||||
return ImageViewKindFromDimension(mem.image_dimension);
|
||||
}
|
||||
|
||||
ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
bool uint_image, uint32_t use_pc) {
|
||||
(void)state;
|
||||
(void)uint_image;
|
||||
(void)use_pc;
|
||||
return ImageViewKindFromDimension(mem.image_dimension);
|
||||
}
|
||||
|
||||
uint32_t ImageViewCoordinateComponents(ImageViewKind view) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim1D: return 1u;
|
||||
case ImageViewKind::Dim1DArray:
|
||||
case ImageViewKind::Dim2D: return 2u;
|
||||
case ImageViewKind::Dim2DArray:
|
||||
case ImageViewKind::Dim2DMsaaArray:
|
||||
case ImageViewKind::Dim3D: return 3u;
|
||||
case ImageViewKind::Dim2DMsaa: return 2u;
|
||||
default: return 0u;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageViewSpatialComponents(ImageViewKind view) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim1D:
|
||||
case ImageViewKind::Dim1DArray: return 1u;
|
||||
case ImageViewKind::Dim2D:
|
||||
case ImageViewKind::Dim2DArray:
|
||||
case ImageViewKind::Dim2DMsaa:
|
||||
case ImageViewKind::Dim2DMsaaArray: return 2u;
|
||||
case ImageViewKind::Dim3D: return 3u;
|
||||
default: return 0u;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) {
|
||||
return state.sampled_images[SampledImageIndex(integer, view)].image_type;
|
||||
}
|
||||
|
||||
uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) {
|
||||
return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type;
|
||||
}
|
||||
|
||||
uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) {
|
||||
switch (ImageViewCoordinateComponents(view)) {
|
||||
case 1u: return state.uint_type;
|
||||
case 2u: return state.vec2_uint_type;
|
||||
case 3u: return state.vec3_uint_type;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
return state.storage_images[StorageImageIndex(uint_image, view)].image_type;
|
||||
}
|
||||
|
||||
uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
return state.storage_images[StorageImageIndex(uint_image, view)].pointer_type;
|
||||
}
|
||||
|
||||
uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
return state.storage_images[StorageImageIndex(uint_image, view)].variable;
|
||||
}
|
||||
|
||||
uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
|
||||
ImageViewKind view) {
|
||||
(void)use_pc;
|
||||
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
|
||||
const auto kind = SampledBindingKind(integer, view);
|
||||
const auto binding = ResourceForDescriptor(state, kind, mem.resource);
|
||||
const auto& descriptors = state.sampled_images[SampledImageIndex(integer, view)];
|
||||
const auto pointer = DescriptorElementPointer(
|
||||
state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
|
||||
mem.resource, "sampled image descriptor array was not emitted");
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
|
||||
(void)use_pc;
|
||||
const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
|
||||
const auto pointer = DescriptorElementPointer(
|
||||
state, state.ptr_uniform_sampler, state.sampler_variable, binding.array_index,
|
||||
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
|
||||
const auto sampler_id = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.sampler_type, sampler_id, pointer});
|
||||
return sampler_id;
|
||||
}
|
||||
|
||||
uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
|
||||
ImageViewKind view) {
|
||||
const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view);
|
||||
const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
|
||||
if (image == 0 || sampler == 0) {
|
||||
ExitDescriptorBindingFailure(
|
||||
state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
|
||||
"sampled image or sampler descriptor load failed");
|
||||
}
|
||||
const auto sampled_image = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSampledImage,
|
||||
ImageViewSampledImageType(state, view, mem.kind == IR::ResourceKind::ImageUint),
|
||||
sampled_image, image, sampler});
|
||||
return sampled_image;
|
||||
}
|
||||
|
||||
uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
|
||||
uint32_t use_pc, ImageViewKind view) {
|
||||
(void)use_pc;
|
||||
const auto kind = StorageBindingKind(uint_image, view);
|
||||
const auto binding = ResourceForDescriptor(state, kind, resource);
|
||||
const auto ptr_type = StorageImagePointerType(state, uint_image, view);
|
||||
const auto variable = StorageImageVariable(state, uint_image, view);
|
||||
return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
|
||||
"storage image descriptor array was not emitted");
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
|
||||
uint32_t use_pc, ImageViewKind view) {
|
||||
const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
|
||||
if (pointer == 0) {
|
||||
ExitDescriptorBindingFailure(state, StorageBindingKind(uint_image, view), resource,
|
||||
"storage image descriptor pointer creation failed");
|
||||
}
|
||||
const auto type = StorageImageType(state, uint_image, view);
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
uint32_t ExecutionModelForStage(ShaderType stage) {
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex: return ExecutionModelVertex;
|
||||
case ShaderType::Pixel: return ExecutionModelFragment;
|
||||
default: return ExecutionModelGLCompute;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,52 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
|
||||
if (reg.file != IR::RegisterFile::Scalar) {
|
||||
return false;
|
||||
}
|
||||
const auto& registers = state.program.bindings.user_data_registers;
|
||||
const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index);
|
||||
if (found == registers.end() || *found != reg.index) {
|
||||
return false;
|
||||
}
|
||||
dword_index = static_cast<uint32_t>(found - registers.begin());
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
|
||||
if (state.push_constant_variable != 0) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
|
||||
state.push_constant_variable, ConstantU32(state, 0),
|
||||
ConstantU32(state, dword_index)});
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
if (state.vsharp_storage_variable != 0) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
|
||||
state.vsharp_storage_variable, ConstantU32(state, 0),
|
||||
ConstantU32(state, dword_index)});
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
|
||||
if (reg.file != IR::RegisterFile::Exec) {
|
||||
return 0;
|
||||
}
|
||||
if (state.per_invocation_masks) {
|
||||
return reg.index == 0 ? 1u : 0u;
|
||||
}
|
||||
return reg.index == 0 || state.wave_size == 64u ? 0xffffffffu : 0u;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,227 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
|
||||
if (!enabled || component >= inst.src_count || component >= 4u) {
|
||||
return ConstantF32(state, component == 3u ? 0x3f800000u : 0u);
|
||||
}
|
||||
return EmitFloatLoad(state, inst.src[component]);
|
||||
}
|
||||
|
||||
uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.compr) {
|
||||
uint32_t components[4] = {
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0x3f800000u),
|
||||
};
|
||||
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
|
||||
pair_index++) {
|
||||
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
|
||||
state.glsl_std450, GlslUnpackHalf2x16, raw});
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair_index * 2u + lane;
|
||||
if (((inst.export_info.en >> component) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
components[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, components[component], unpacked, lane});
|
||||
}
|
||||
}
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, components[0],
|
||||
components[1], components[2], components[3]});
|
||||
return vec;
|
||||
}
|
||||
|
||||
const auto x = EmitExportComponentF32(state, inst, 0);
|
||||
const auto y = EmitExportComponentF32(state, inst, 1);
|
||||
const auto z = EmitExportComponentF32(state, inst, 2);
|
||||
const auto w = EmitExportComponentF32(state, inst, 3);
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w});
|
||||
return vec;
|
||||
}
|
||||
|
||||
uint32_t EmitExportComponentU32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
|
||||
if (!enabled || component >= inst.src_count || component >= 4u) {
|
||||
return ConstantU32(state, component == 3u ? 1u : 0u);
|
||||
}
|
||||
return EmitValueLoad(state, inst.src[component]);
|
||||
}
|
||||
|
||||
uint32_t EmitExportVec4U32(EmitterState& state, const IR::Instruction& inst) {
|
||||
uint32_t components[4] = {
|
||||
ConstantU32(state, 0u),
|
||||
ConstantU32(state, 0u),
|
||||
ConstantU32(state, 0u),
|
||||
ConstantU32(state, 1u),
|
||||
};
|
||||
|
||||
if (inst.export_info.compr) {
|
||||
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
|
||||
pair_index++) {
|
||||
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair_index * 2u + lane;
|
||||
if (((inst.export_info.en >> component) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
components[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitFieldUExtract, state.uint_type, components[component], raw,
|
||||
ConstantU32(state, lane * 16u), ConstantU32(state, 16u)});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (uint32_t component = 0; component < 4u; component++) {
|
||||
components[component] = EmitExportComponentU32(state, inst, component);
|
||||
}
|
||||
}
|
||||
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, vec, components[0],
|
||||
components[1], components[2], components[3]});
|
||||
return vec;
|
||||
}
|
||||
|
||||
static bool MrtUsesUintOutput(const EmitterState& state, const IR::Instruction& inst) {
|
||||
return inst.export_info.kind == IR::ExportTargetKind::Mrt &&
|
||||
state.pixel_input_info != nullptr &&
|
||||
inst.export_info.index < std::size(state.pixel_input_info->target_output_mode) &&
|
||||
state.pixel_input_info->target_output_mode[inst.export_info.index] == 7u;
|
||||
}
|
||||
|
||||
uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value,
|
||||
uint32_t vector_type) {
|
||||
if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state.pixel_input_info == nullptr ||
|
||||
inst.export_info.index >= state.pixel_input_info->target_export_mapping.size()) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto mapping = state.pixel_input_info->target_export_mapping[inst.export_info.index];
|
||||
if (mapping.IsIdentity()) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto mapped = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value, mapping.Map(0),
|
||||
mapping.Map(1), mapping.Map(2), mapping.Map(3)});
|
||||
return mapped;
|
||||
}
|
||||
|
||||
bool ExportWritesData(const IR::Instruction& inst) {
|
||||
switch (inst.export_info.kind) {
|
||||
case IR::ExportTargetKind::Null:
|
||||
case IR::ExportTargetKind::Primitive: return false;
|
||||
case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0;
|
||||
default: return inst.export_info.en != 0;
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) {
|
||||
if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) {
|
||||
state.builder.AddFunction(
|
||||
{OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)});
|
||||
}
|
||||
|
||||
if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) {
|
||||
const auto raw =
|
||||
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto ptr = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, mask, raw});
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr,
|
||||
state.sample_mask_variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, ptr, mask});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitExport(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
|
||||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ExportWritesData(inst)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) {
|
||||
EmitMrtZExport(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto variable = OutputVariableForExport(state, inst.export_info);
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto uint_output = MrtUsesUintOutput(state, inst);
|
||||
const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
|
||||
const auto value = ApplyMrtExportMapping(
|
||||
state, inst, uint_output ? EmitExportVec4U32(state, inst) : EmitExportVec4F32(state, inst),
|
||||
vector_type);
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_output_vec4_float, pointer, variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, pointer, value});
|
||||
return;
|
||||
}
|
||||
|
||||
state.builder.AddFunction({OpStore, variable, value});
|
||||
}
|
||||
|
||||
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
|
||||
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
|
||||
}
|
||||
|
||||
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
|
||||
const auto kill_label = state.builder.AllocateId();
|
||||
const auto merge_label = state.builder.AllocateId();
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
|
||||
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
|
||||
state.builder.AddFunction({OpLabel, kill_label});
|
||||
state.builder.AddFunction({OpKill});
|
||||
state.builder.AddFunction({OpLabel, merge_label});
|
||||
}
|
||||
|
||||
void EmitUpdatePixelValidMask(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
|
||||
}
|
||||
|
||||
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto mask_value = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
|
||||
EmitKillIfBoolFalse(state, active);
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,379 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitFloatLoad(state, inst.src[0]);
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
|
||||
uint32_t limit = base.index + 1u;
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg.file == IR::RegisterFile::Vector) {
|
||||
limit = std::max(limit, binding.reg.index + 1u);
|
||||
}
|
||||
}
|
||||
return std::min(limit, 256u);
|
||||
}
|
||||
|
||||
void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.src[0].kind != IR::OperandKind::Register ||
|
||||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.src[0].reg;
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
auto selected = EmitRegisterLoad(state, base_reg);
|
||||
const auto limit = MoveRelRegisterLimit(state, base_reg);
|
||||
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
|
||||
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
|
||||
const auto match = state.builder.AllocateId();
|
||||
const auto next = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected});
|
||||
selected = next;
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, selected);
|
||||
}
|
||||
|
||||
void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.dst.kind != IR::OperandKind::Register ||
|
||||
inst.dst.reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.dst.reg;
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
const auto exec_active = EmitExecActiveBool(state);
|
||||
const auto limit = MoveRelRegisterLimit(state, base_reg);
|
||||
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
|
||||
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg});
|
||||
if (pointer == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto old_value = state.builder.AllocateId();
|
||||
const auto match = state.builder.AllocateId();
|
||||
const auto write = state.builder.AllocateId();
|
||||
const auto merged = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value});
|
||||
state.builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
IsMaskRegisterFile(inst.dst.reg.file)) {
|
||||
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
|
||||
return;
|
||||
}
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
EmitStoreU32(state, inst.dst, low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
|
||||
}
|
||||
|
||||
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
|
||||
uint32_t ret = ConstantU32(state, 0);
|
||||
for (uint32_t i = 0; i < 8u; i++) {
|
||||
const auto mask = 0x0fu << (i * 4u);
|
||||
const auto masked = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto expanded = state.builder.AllocateId();
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
|
||||
ConstantU32(state, mask), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
|
||||
ret = combined;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file == IR::RegisterFile::Scalar) {
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
const auto ret_low = EmitWqmLaneU32(state, low);
|
||||
const auto ret_high = EmitWqmLaneU32(state, high);
|
||||
EmitStoreU32(state, inst.dst, ret_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
|
||||
return;
|
||||
}
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup),
|
||||
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
|
||||
const auto low = state.builder.AllocateId();
|
||||
const auto high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
|
||||
const auto wqm_low = EmitWqmLaneU32(state, low);
|
||||
const auto wqm_high = EmitWqmLaneU32(state, high);
|
||||
const auto lane = EmitSubgroupLocalInvocationId(state);
|
||||
const auto high_half = state.builder.AllocateId();
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto bit_index = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
if (state.per_invocation_masks) {
|
||||
EmitPerInvocationMask(state, inst.dst, active);
|
||||
} else {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, result, active, ConstantU32(state, 1),
|
||||
ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, inst.dst, result);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto old_exec = EmitExecActiveBool(state);
|
||||
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
|
||||
EmitPerInvocationMask(state, inst.dst, old_exec);
|
||||
|
||||
uint32_t lhs = old_exec;
|
||||
uint32_t rhs = src;
|
||||
bool use_or = false;
|
||||
switch (inst.saveexec_mode) {
|
||||
case IR::SaveexecMode::And: break;
|
||||
case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break;
|
||||
case IR::SaveexecMode::Orn2:
|
||||
lhs = EmitLogicalNotBool(state, old_exec);
|
||||
use_or = true;
|
||||
break;
|
||||
}
|
||||
const auto result =
|
||||
use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs);
|
||||
EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result);
|
||||
EmitStoreSccBool(state, result);
|
||||
}
|
||||
|
||||
void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto src_low = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
|
||||
uint32_t rhs_low = src_low;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
|
||||
}
|
||||
|
||||
const auto new_low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
|
||||
if (state.wave_size == 32u) {
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
|
||||
}
|
||||
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto scc = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high);
|
||||
|
||||
uint32_t lhs_low = old_low;
|
||||
uint32_t lhs_high = old_high;
|
||||
uint32_t rhs_low = src_low;
|
||||
uint32_t rhs_high = src_high;
|
||||
uint32_t opcode = OpBitwiseAnd;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state.builder.AllocateId();
|
||||
rhs_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high});
|
||||
}
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
|
||||
lhs_low = state.builder.AllocateId();
|
||||
lhs_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low});
|
||||
state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high});
|
||||
opcode = OpBitwiseOr;
|
||||
}
|
||||
|
||||
const auto new_low = state.builder.AllocateId();
|
||||
const auto new_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low});
|
||||
state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
|
||||
|
||||
const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high;
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto scc = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
const auto first_lane = state.builder.AllocateId();
|
||||
const auto first_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), active});
|
||||
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane,
|
||||
ConstantU32(state, ScopeSubgroup), ballot});
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value,
|
||||
ConstantU32(state, ScopeSubgroup), src, first_lane});
|
||||
EmitStoreU32(state, inst.dst, first_value);
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto mask = state.wave_size == 32u ? 31u : 63u;
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand),
|
||||
ConstantU32(state, mask)});
|
||||
return lane;
|
||||
}
|
||||
|
||||
void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
|
||||
ConstantU32(state, ScopeSubgroup), src, lane});
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto pointer =
|
||||
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0;
|
||||
const auto old = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0);
|
||||
const auto merged = state.builder.AllocateId();
|
||||
if (pointer != 0) {
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, old, pointer});
|
||||
}
|
||||
state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old});
|
||||
if (pointer != 0) {
|
||||
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
|
||||
state.builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane8 = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto upper = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto index0 = state.builder.AllocateId();
|
||||
const auto index1 = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto src1 = EmitValueLoad(state, inst.src[1]);
|
||||
const auto src2 = EmitValueLoad(state, inst.src[2]);
|
||||
uint32_t row_value = row;
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
if (x16) {
|
||||
row_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
|
||||
}
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1});
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target});
|
||||
uint32_t ret = shuffled;
|
||||
if (!inst.dst.op_sel) {
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target);
|
||||
ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, ret);
|
||||
}
|
||||
|
||||
void EmitControlNop(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitBarrier(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)inst;
|
||||
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
|
||||
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
|
||||
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,333 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t EmitCubeAxisF32(EmitterState& state, uint32_t value) {
|
||||
const auto normalized = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpFSub, state.float_type, normalized, value, ConstantF32(state, 0x3f800000u)});
|
||||
return normalized;
|
||||
}
|
||||
|
||||
uint32_t EmitCubeLayerF32(EmitterState& state, uint32_t face_id) {
|
||||
// Sampled RDNA2 cubemaps encode face_id as slice * 8 + face. The native
|
||||
// 2D-array view stores six contiguous faces per slice, so remove the two
|
||||
// reserved face IDs from every preceding slice.
|
||||
const auto guest_layer = state.builder.AllocateId();
|
||||
const auto slice = state.builder.AllocateId();
|
||||
const auto padding = state.builder.AllocateId();
|
||||
const auto host_layer = state.builder.AllocateId();
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertFToU, state.uint_type, guest_layer, face_id});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, slice, guest_layer, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, padding, slice, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, host_layer, guest_layer, padding});
|
||||
state.builder.AddFunction({OpConvertUToF, state.float_type, result, host_layer});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordF32Impl(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, uint32_t first_component,
|
||||
uint32_t components) {
|
||||
auto x = EmitImageAddressFloatLoad(state, inst, address, first_component);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
auto y = inst.memory.image_address_components > first_component + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, address, first_component + 1u)
|
||||
: EmitZeroF32(state);
|
||||
if (inst.memory.image_cube) {
|
||||
// RDNA2 sampled cubemap S/T coordinates are biased by +1 relative to
|
||||
// normalized 2D-array coordinates.
|
||||
x = EmitCubeAxisF32(state, x);
|
||||
y = EmitCubeAxisF32(state, y);
|
||||
}
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
auto z = inst.memory.image_address_components > first_component + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, address, first_component + 2u)
|
||||
: EmitZeroF32(state);
|
||||
if (inst.memory.image_cube) {
|
||||
z = EmitCubeLayerF32(state, z);
|
||||
}
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
|
||||
return (inst.memory.image_sample_flags & flag) != 0;
|
||||
}
|
||||
|
||||
ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) {
|
||||
ImageSampleLayout layout;
|
||||
uint32_t cursor = 0;
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
layout.offset = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
layout.bias = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) {
|
||||
layout.dref = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
layout.grad_x = cursor;
|
||||
cursor += components;
|
||||
layout.grad_y = cursor;
|
||||
cursor += components;
|
||||
}
|
||||
layout.coord = cursor;
|
||||
cursor += ImageViewCoordinateComponents(view);
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) {
|
||||
layout.lod = cursor++;
|
||||
}
|
||||
return layout;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
return EmitImageCoordF32Impl(state, inst, inst.src[0], layout.coord,
|
||||
ImageViewCoordinateComponents(view));
|
||||
}
|
||||
|
||||
uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
|
||||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
|
||||
}
|
||||
|
||||
uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
|
||||
}
|
||||
|
||||
uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
|
||||
}
|
||||
|
||||
uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t first_component, ImageViewKind view) {
|
||||
const auto x = inst.memory.image_address_components > first_component
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
|
||||
: EmitZeroF32(state);
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > first_component + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
|
||||
: EmitZeroF32(state);
|
||||
const auto grad = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z =
|
||||
inst.memory.image_address_components > first_component + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 2u)
|
||||
: EmitZeroF32(state);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
|
||||
}
|
||||
return grad;
|
||||
}
|
||||
|
||||
uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
if (layout.offset == NoImageComponent ||
|
||||
inst.memory.image_address_components <= layout.offset) {
|
||||
const auto zero = ConstantI32(state, 0);
|
||||
if (components == 1u) {
|
||||
return zero;
|
||||
}
|
||||
const auto ret = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
|
||||
const auto packed_i32 = state.builder.AllocateId();
|
||||
const auto offset_x = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
|
||||
ConstantI32(state, 0), ConstantI32(state, 6)});
|
||||
if (components == 1u) {
|
||||
return offset_x;
|
||||
}
|
||||
const auto offset_y = state.builder.AllocateId();
|
||||
const auto offset = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
|
||||
ConstantI32(state, 8), ConstantI32(state, 6)});
|
||||
if (components == 3u) {
|
||||
const auto offset_z = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32,
|
||||
ConstantI32(state, 16), ConstantI32(state, 6)});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y});
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, ImageViewKind view) {
|
||||
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto lod_component = ImageViewCoordinateComponents(view);
|
||||
if (inst.memory.image_address_components <= lod_component) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitImageAddressValueLoad(state, inst, address, lod_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
// OpImageQueryLod takes only the spatial coordinates, even for arrayed images.
|
||||
return EmitImageCoordF32Impl(state, inst, inst.src[0], 0, ImageViewSpatialComponents(view));
|
||||
}
|
||||
|
||||
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
|
||||
uint32_t index = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
index += (dmask >> i) & 1u;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
|
||||
const auto target = 4u + component;
|
||||
for (uint32_t source = 0; source < 4u; source++) {
|
||||
if (((swizzle >> (source * 3u)) & 0x7u) == target) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.memory.resource >= state.program.info.images.size()) {
|
||||
EXIT("storage image instruction has no specialized swizzle\n");
|
||||
}
|
||||
return state.program.info.images[inst.memory.resource].storage_swizzle;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
|
||||
: ConstantU32(state, 0);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,342 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
namespace {
|
||||
|
||||
uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x),
|
||||
ConstantI32(state, y)});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state, ImageViewKind view) {
|
||||
if (ImageViewSpatialComponents(view) == 1u) {
|
||||
const auto type = state.builder.AllocateId();
|
||||
state.builder.AddType({OpTypeArray, type, state.int_type, ConstantU32(state, 4)});
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, type, value, ConstantI32(state, -1),
|
||||
ConstantI32(state, 0), ConstantI32(state, 1),
|
||||
ConstantI32(state, 2)});
|
||||
return value;
|
||||
}
|
||||
const auto offset0 = ConstantVec2I32(state, -1, 0);
|
||||
const auto offset1 = ConstantVec2I32(state, 0, 0);
|
||||
const auto offset2 = ConstantVec2I32(state, 1, 0);
|
||||
const auto offset3 = ConstantVec2I32(state, 2, 0);
|
||||
const auto type = state.builder.AllocateId();
|
||||
state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)});
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
|
||||
uint32_t array_index, bool uint_image,
|
||||
ImageViewKind view) {
|
||||
const auto kind = StorageBindingKind(uint_image, view);
|
||||
const auto& descriptors = state.storage_images[StorageImageIndex(uint_image, view)];
|
||||
const auto pointer =
|
||||
DescriptorElementPointer(state, descriptors.pointer_type, descriptors.variable, array_index,
|
||||
kind, resource, "storage image descriptor array was not emitted");
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, descriptors.image_type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
|
||||
uint32_t component_index, ImageViewKind view) {
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
switch (component_index) {
|
||||
case 0: {
|
||||
if (components == 1u) {
|
||||
return size;
|
||||
}
|
||||
const auto width = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0});
|
||||
return width;
|
||||
}
|
||||
case 1: {
|
||||
if (components < 2u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto height = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1});
|
||||
return height;
|
||||
}
|
||||
case 2:
|
||||
if (components == 3u) {
|
||||
const auto depth = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2});
|
||||
return depth;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
case 3: {
|
||||
const auto levels = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image});
|
||||
return levels;
|
||||
}
|
||||
default: return ConstantU32(state, 0);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const auto mip_level = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
const auto size = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component =
|
||||
EmitImageGetResinfoComponent(state, image, size, component_index, view);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto lod = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image,
|
||||
EmitImageQueryCoordF32(state, inst, view)});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 2u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, lod, component_index});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
|
||||
const auto color = state.builder.AllocateId();
|
||||
const auto coord = EmitImageLoadCoordU32(state, inst, view);
|
||||
if (ImageSpirvMultisampled(view) != 0) {
|
||||
const auto sample = EmitImageAddressValueLoad(state, inst, inst.src[0],
|
||||
ImageViewCoordinateComponents(view));
|
||||
state.builder.AddFunction({OpImageFetch,
|
||||
integer ? state.vec4_uint_type : state.vec4_float_type, color,
|
||||
image, coord, ImageOperandsSampleMask, sample});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type, color, image,
|
||||
coord, ImageOperandsLodMask, EmitImageMipLodU32(state, inst, inst.src[0], view)});
|
||||
}
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, color, component_index});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
|
||||
const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
|
||||
const auto binding =
|
||||
ResourceForDescriptor(state, StorageBindingKind(uint_image, view), inst.memory.resource);
|
||||
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
|
||||
binding.array_index, uint_image, view);
|
||||
|
||||
state.builder.AddFunction(
|
||||
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
|
||||
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
|
||||
}
|
||||
|
||||
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
|
||||
bool dref, bool integer) {
|
||||
if (dref) {
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample});
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, sample, component_index});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) {
|
||||
// RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it
|
||||
// to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information.
|
||||
// SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need
|
||||
// an explicit operand even for the unsuffixed sample opcode.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) {
|
||||
return true;
|
||||
}
|
||||
return state.stage != ShaderType::Pixel;
|
||||
}
|
||||
|
||||
void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, bool explicit_lod,
|
||||
std::vector<uint32_t>& words) {
|
||||
uint32_t mask = 0;
|
||||
std::vector<uint32_t> operands;
|
||||
|
||||
// Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod
|
||||
// operands belong to explicit-lod instructions; bias belongs to implicit-lod
|
||||
// sampling because it modifies the hardware-computed LOD.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
mask |= ImageOperandsGradMask;
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x, view));
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y, view));
|
||||
} else if (explicit_lod) {
|
||||
mask |= ImageOperandsLodMask;
|
||||
operands.push_back(EmitImageLodF32(state, inst, layout));
|
||||
} else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
mask |= ImageOperandsBiasMask;
|
||||
operands.push_back(EmitImageBiasF32(state, inst, layout));
|
||||
}
|
||||
if (mask != 0) {
|
||||
words.push_back(mask);
|
||||
words.insert(words.end(), operands.begin(), operands.end());
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto flags = inst.memory.image_sample_flags;
|
||||
const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0;
|
||||
const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
if (explicit_lod) {
|
||||
return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
|
||||
}
|
||||
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
|
||||
}
|
||||
|
||||
void EmitImageSample(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto sample = state.builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
uint32_t result_type = state.vec4_float_type;
|
||||
if (dref) {
|
||||
result_type = state.float_type;
|
||||
} else if (integer) {
|
||||
result_type = state.vec4_uint_type;
|
||||
}
|
||||
const auto explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
const auto opcode = ImageSampleOpcode(state, inst);
|
||||
std::vector<uint32_t> words = {opcode, result_type, sample, sampled_image, base_coord};
|
||||
if (dref) {
|
||||
words.push_back(EmitImageDrefF32(state, inst, layout));
|
||||
}
|
||||
AddImageSampleOperands(state, inst, layout, explicit_lod, words);
|
||||
state.builder.AddFunction(words);
|
||||
EmitImageSampleResult(state, inst, sample, dref, integer);
|
||||
}
|
||||
|
||||
uint32_t ImageGatherComponent(uint32_t dmask) {
|
||||
switch (dmask) {
|
||||
case 0x2u: return 1;
|
||||
case 0x4u: return 2;
|
||||
case 0x8u: return 3;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view,
|
||||
std::vector<uint32_t>& words) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
|
||||
words.push_back(ImageOperandsConstOffsetsMask);
|
||||
words.push_back(ConstantImageGatherHorizontalOffsets(state, view));
|
||||
return;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
words.push_back(ImageOperandsOffsetMask);
|
||||
words.push_back(EmitImagePackedOffsetI32(state, inst, layout, view));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto texels = state.builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
const auto result_type = integer ? state.vec4_uint_type : state.vec4_float_type;
|
||||
std::vector<uint32_t> words =
|
||||
dref ? std::vector<uint32_t> {OpImageDrefGather,
|
||||
state.vec4_float_type,
|
||||
texels,
|
||||
sampled_image,
|
||||
coord,
|
||||
EmitImageDrefF32(state, inst, layout)}
|
||||
: std::vector<uint32_t> {
|
||||
OpImageGather, result_type,
|
||||
texels, sampled_image,
|
||||
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
|
||||
AddImageGatherOperands(state, inst, layout, view, words);
|
||||
state.builder.AddFunction(words);
|
||||
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, texels, i});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,853 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind) {
|
||||
return IR::FindBinding(state.program.bindings, kind);
|
||||
}
|
||||
|
||||
uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind) {
|
||||
const auto* binding = DescriptorBinding(state, kind);
|
||||
return binding != nullptr ? static_cast<uint32_t>(binding->resources.size()) : 0;
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState& state, uint32_t value) {
|
||||
if (auto it = state.constants.find(value); it != state.constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.constants.emplace(value, id);
|
||||
state.builder.AddType({OpConstant, state.uint_type, id, value});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantI32(EmitterState& state, int32_t value) {
|
||||
const auto bits = static_cast<uint32_t>(value);
|
||||
if (auto it = state.signed_constants.find(bits); it != state.signed_constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.signed_constants.emplace(bits, id);
|
||||
state.builder.AddType({OpConstant, state.int_type, id, bits});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantF32(EmitterState& state, uint32_t bits) {
|
||||
if (auto it = state.float_constants.find(bits); it != state.float_constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.float_constants.emplace(bits, id);
|
||||
state.builder.AddType({OpConstant, state.float_type, id, bits});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t FloatBits(float value) {
|
||||
uint32_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
return bits;
|
||||
}
|
||||
|
||||
uint32_t ConstantF32Value(EmitterState& state, float value) {
|
||||
return ConstantF32(state, FloatBits(value));
|
||||
}
|
||||
|
||||
VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
|
||||
if (state.stage != ShaderType::Vertex || state.vertex_input_info == nullptr ||
|
||||
location >= ShaderVertexInputInfo::RES_MAX ||
|
||||
location >= static_cast<uint32_t>(state.vertex_input_info->resources_num)) {
|
||||
return VertexInputScalarKind::Float;
|
||||
}
|
||||
|
||||
switch (state.vertex_input_info->resources[location].Format()) {
|
||||
case Prospero::BufferFormat::k8UInt:
|
||||
case Prospero::BufferFormat::k16UInt:
|
||||
case Prospero::BufferFormat::k8_8UInt:
|
||||
case Prospero::BufferFormat::k32UInt:
|
||||
case Prospero::BufferFormat::k16_16UInt:
|
||||
case Prospero::BufferFormat::k8_8_8_8UInt:
|
||||
case Prospero::BufferFormat::k32_32UInt:
|
||||
case Prospero::BufferFormat::k16_16_16_16UInt:
|
||||
case Prospero::BufferFormat::k32_32_32UInt:
|
||||
case Prospero::BufferFormat::k32_32_32_32UInt: return VertexInputScalarKind::Uint;
|
||||
case Prospero::BufferFormat::k8SInt:
|
||||
case Prospero::BufferFormat::k16SInt:
|
||||
case Prospero::BufferFormat::k8_8SInt:
|
||||
case Prospero::BufferFormat::k32SInt:
|
||||
case Prospero::BufferFormat::k16_16SInt:
|
||||
case Prospero::BufferFormat::k8_8_8_8SInt:
|
||||
case Prospero::BufferFormat::k32_32SInt:
|
||||
case Prospero::BufferFormat::k16_16_16_16SInt:
|
||||
case Prospero::BufferFormat::k32_32_32SInt:
|
||||
case Prospero::BufferFormat::k32_32_32_32SInt: return VertexInputScalarKind::Sint;
|
||||
default: return VertexInputScalarKind::Float;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) {
|
||||
uint32_t count = input.component_count;
|
||||
if (state.stage == ShaderType::Vertex && state.vertex_input_info != nullptr &&
|
||||
input.location < ShaderVertexInputInfo::RES_MAX &&
|
||||
input.location < static_cast<uint32_t>(state.vertex_input_info->resources_num) &&
|
||||
state.vertex_input_info->resources_dst[input.location].registers_num > 0) {
|
||||
count = static_cast<uint32_t>(
|
||||
state.vertex_input_info->resources_dst[input.location].registers_num);
|
||||
}
|
||||
return std::clamp(count, 1u, 4u);
|
||||
}
|
||||
|
||||
uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint: return state.int_type;
|
||||
case VertexInputScalarKind::Uint: return state.uint_type;
|
||||
case VertexInputScalarKind::Float:
|
||||
default: return state.float_type;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint: return state.ptr_input_int;
|
||||
case VertexInputScalarKind::Uint: return state.ptr_input_uint;
|
||||
case VertexInputScalarKind::Float:
|
||||
default: return state.ptr_input_float;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t components) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint:
|
||||
switch (components) {
|
||||
case 1: return state.int_type;
|
||||
case 2: return state.vec2_int_type;
|
||||
case 3: return state.vec3_int_type;
|
||||
default: return state.vec4_int_type;
|
||||
}
|
||||
case VertexInputScalarKind::Uint:
|
||||
switch (components) {
|
||||
case 1: return state.uint_type;
|
||||
case 2: return state.vec2_uint_type;
|
||||
case 3: return state.vec3_uint_type;
|
||||
default: return state.vec4_uint_type;
|
||||
}
|
||||
case VertexInputScalarKind::Float:
|
||||
default:
|
||||
switch (components) {
|
||||
case 1: return state.float_type;
|
||||
case 2: return state.vec2_float_type;
|
||||
case 3: return state.vec3_float_type;
|
||||
default: return state.vec4_float_type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t components) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_int;
|
||||
case 2: return state.ptr_input_vec2_int;
|
||||
case 3: return state.ptr_input_vec3_int;
|
||||
default: return state.ptr_input_vec4_int;
|
||||
}
|
||||
case VertexInputScalarKind::Uint:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_uint;
|
||||
case 2: return state.ptr_input_vec2_uint;
|
||||
case 3: return state.ptr_input_vec3_uint;
|
||||
default: return state.ptr_input_vec4_uint;
|
||||
}
|
||||
case VertexInputScalarKind::Float:
|
||||
default:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_float;
|
||||
case 2: return state.ptr_input_vec2_float;
|
||||
case 3: return state.ptr_input_vec3_float;
|
||||
default: return state.ptr_input_vec4_float;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) {
|
||||
return state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr &&
|
||||
index < std::size(state.pixel_input_info->target_output_mode) &&
|
||||
state.pixel_input_info->target_output_mode[index] == 7u;
|
||||
}
|
||||
|
||||
void AllocateInputVariables(EmitterState& state) {
|
||||
for (auto& binding: state.inputs) {
|
||||
binding.variable_id = state.builder.AllocateId();
|
||||
state.interface_variables.push_back(binding.variable_id);
|
||||
}
|
||||
if (state.needs_subgroup_local_invocation_id) {
|
||||
state.subgroup_local_invocation_id_variable = state.builder.AllocateId();
|
||||
state.interface_variables.push_back(state.subgroup_local_invocation_id_variable);
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t AllocateInterfaceVariable(EmitterState& state) {
|
||||
const auto variable = state.builder.AllocateId();
|
||||
state.interface_variables.push_back(variable);
|
||||
return variable;
|
||||
}
|
||||
|
||||
static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& variable) {
|
||||
if (variable == 0) {
|
||||
variable = AllocateInterfaceVariable(state);
|
||||
}
|
||||
return variable;
|
||||
}
|
||||
|
||||
void AllocateOutputVariables(EmitterState& state) {
|
||||
for (auto& binding: state.outputs) {
|
||||
switch (binding.kind) {
|
||||
case IR::StageOutputKind::Position:
|
||||
binding.variable_id =
|
||||
AllocateSharedOutputVariable(state, state.per_vertex_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::Depth:
|
||||
binding.variable_id = AllocateSharedOutputVariable(state, state.depth_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::SampleMask:
|
||||
binding.variable_id =
|
||||
AllocateSharedOutputVariable(state, state.sample_mask_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::Parameter:
|
||||
case IR::StageOutputKind::Mrt:
|
||||
binding.variable_id = AllocateInterfaceVariable(state);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t BuiltInForInput(IR::StageInputKind kind) {
|
||||
switch (kind) {
|
||||
case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex;
|
||||
case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex;
|
||||
case IR::StageInputKind::FragCoord: return BuiltInFragCoord;
|
||||
case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing;
|
||||
case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId;
|
||||
case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId;
|
||||
case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex;
|
||||
case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId;
|
||||
default: return UINT32_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
void AddInputAnnotationsAndNames(EmitterState& state) {
|
||||
if (state.subgroup_local_invocation_id_variable != 0) {
|
||||
state.builder.AddName(state.subgroup_local_invocation_id_variable,
|
||||
"gl_SubgroupInvocationID");
|
||||
state.builder.AddAnnotation({OpDecorate, state.subgroup_local_invocation_id_variable,
|
||||
DecorationBuiltIn, BuiltInSubgroupLocalInvocationId});
|
||||
if (state.stage == ShaderType::Pixel) {
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.subgroup_local_invocation_id_variable, DecorationFlat});
|
||||
}
|
||||
}
|
||||
for (const auto& input: state.inputs) {
|
||||
state.builder.AddName(input.variable_id, input.debug_name.c_str());
|
||||
if (input.kind == IR::StageInputKind::Parameter) {
|
||||
const auto flat = PixelParameterIsFlat(state, input.location);
|
||||
if (flat) {
|
||||
state.builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
|
||||
}
|
||||
if (state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr &&
|
||||
state.pixel_input_info->ps_no_perspective && !flat) {
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationNoPerspective});
|
||||
}
|
||||
const auto location = PixelParameterLocation(state, input.location);
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationLocation, location});
|
||||
continue;
|
||||
}
|
||||
const auto builtin = BuiltInForInput(input.kind);
|
||||
if (builtin != UINT32_MAX) {
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationBuiltIn, builtin});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AddOutputAnnotationsAndNames(EmitterState& state) {
|
||||
if (state.per_vertex_variable != 0) {
|
||||
state.builder.AddName(state.per_vertex_type, "gl_PerVertex");
|
||||
state.builder.AddName(state.per_vertex_variable, "outPerVertex");
|
||||
state.builder.AddAnnotation(
|
||||
{OpMemberDecorate, state.per_vertex_type, 0, DecorationBuiltIn, BuiltInPosition});
|
||||
state.builder.AddAnnotation({OpDecorate, state.per_vertex_type, DecorationBlock});
|
||||
}
|
||||
if (state.depth_variable != 0) {
|
||||
state.builder.AddName(state.depth_variable, "gl_FragDepth");
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.depth_variable, DecorationBuiltIn, BuiltInFragDepth});
|
||||
}
|
||||
if (state.sample_mask_variable != 0) {
|
||||
state.builder.AddName(state.sample_mask_variable, "gl_SampleMask");
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask});
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
if (binding.kind == IR::StageOutputKind::Parameter ||
|
||||
binding.kind == IR::StageOutputKind::Mrt) {
|
||||
state.builder.AddName(binding.variable_id, binding.debug_name.c_str());
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, binding.variable_id, DecorationLocation, binding.location});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, uint32_t set,
|
||||
uint32_t binding) {
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
state.builder.AddName(variable, name);
|
||||
state.builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, set});
|
||||
state.builder.AddAnnotation({OpDecorate, variable, DecorationBinding, binding});
|
||||
}
|
||||
|
||||
void AddDescriptorAnnotationsAndNames(EmitterState& state) {
|
||||
const auto set = state.program.bindings.descriptor_set;
|
||||
auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) {
|
||||
const auto* binding = DescriptorBinding(state, kind);
|
||||
if (binding != nullptr) {
|
||||
DecorateDescriptor(state, variable, name, set, binding->binding);
|
||||
}
|
||||
};
|
||||
if (state.storage_buffer_variable != 0) {
|
||||
Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
|
||||
}
|
||||
if (state.address_memory_variable != 0) {
|
||||
Decorate(state.address_memory_variable, "address_memory",
|
||||
IR::DescriptorBindingKind::AddressMemory);
|
||||
}
|
||||
constexpr const char* SampledNames[] = {"sampled_1d",
|
||||
"sampled_1d_array",
|
||||
"sampled_2d",
|
||||
"sampled_2d_array",
|
||||
"sampled_3d",
|
||||
"sampled_2d_msaa",
|
||||
"sampled_2d_msaa_array",
|
||||
"sampled_uint_1d",
|
||||
"sampled_uint_1d_array",
|
||||
"sampled_uint_2d",
|
||||
"sampled_uint_2d_array",
|
||||
"sampled_uint_3d",
|
||||
"sampled_uint_2d_msaa",
|
||||
"sampled_uint_2d_msaa_array"};
|
||||
for (uint32_t i = 0; i < state.sampled_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % SampledImageViewKindCount);
|
||||
Decorate(state.sampled_images[i].variable, SampledNames[i],
|
||||
SampledBindingKind(i >= SampledImageViewKindCount, view));
|
||||
}
|
||||
constexpr const char* StorageNames[] = {"storage_1d",
|
||||
"storage_1d_array",
|
||||
"storage_2d",
|
||||
"storage_2d_array",
|
||||
"storage_3d",
|
||||
"storage_uint_1d",
|
||||
"storage_uint_1d_array",
|
||||
"storage_uint_2d",
|
||||
"storage_uint_2d_array",
|
||||
"storage_uint_3d"};
|
||||
for (uint32_t i = 0; i < state.storage_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % StorageImageViewKindCount);
|
||||
Decorate(state.storage_images[i].variable, StorageNames[i],
|
||||
StorageBindingKind(i >= StorageImageViewKindCount, view));
|
||||
}
|
||||
if (state.sampler_variable != 0) {
|
||||
Decorate(state.sampler_variable, "samplers", IR::DescriptorBindingKind::Samplers);
|
||||
}
|
||||
if (state.gds_variable != 0) {
|
||||
Decorate(state.gds_variable, "gds", IR::DescriptorBindingKind::Gds);
|
||||
}
|
||||
if (state.flattened_srt_variable != 0) {
|
||||
Decorate(state.flattened_srt_variable, "flattened_srt",
|
||||
IR::DescriptorBindingKind::FlattenedSrt);
|
||||
}
|
||||
}
|
||||
|
||||
void AddVsharpAnnotationsAndNames(EmitterState& state) {
|
||||
const auto& bind = state.program.bindings;
|
||||
if (state.push_constant_variable != 0) {
|
||||
state.builder.AddName(state.push_constant_block_type, "BufferResource");
|
||||
state.builder.AddName(state.push_constant_variable, "vsharp");
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.push_constant_array_type, DecorationArrayStride, 4});
|
||||
state.builder.AddAnnotation({OpMemberDecorate, state.push_constant_block_type, 0,
|
||||
DecorationOffset, bind.push_constant_offset});
|
||||
state.builder.AddAnnotation({OpDecorate, state.push_constant_block_type, DecorationBlock});
|
||||
}
|
||||
if (state.vsharp_storage_variable != 0) {
|
||||
const auto* storage = DescriptorBinding(state, IR::DescriptorBindingKind::UserData);
|
||||
if (storage != nullptr) {
|
||||
DecorateDescriptor(state, state.vsharp_storage_variable, "user_data",
|
||||
bind.descriptor_set, storage->binding);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void EmitHeaderAndTypes(EmitterState& state) {
|
||||
state.void_type = state.builder.AllocateId();
|
||||
state.bool_type = state.builder.AllocateId();
|
||||
state.uint_type = state.builder.AllocateId();
|
||||
state.uint_pair_type = state.builder.AllocateId();
|
||||
state.int_type = state.builder.AllocateId();
|
||||
state.int_pair_type = state.builder.AllocateId();
|
||||
state.float_type = state.builder.AllocateId();
|
||||
state.vec2_uint_type = state.builder.AllocateId();
|
||||
state.vec3_uint_type = state.builder.AllocateId();
|
||||
state.vec4_uint_type = state.builder.AllocateId();
|
||||
state.vec2_int_type = state.builder.AllocateId();
|
||||
state.vec3_int_type = state.builder.AllocateId();
|
||||
state.vec4_int_type = state.builder.AllocateId();
|
||||
state.vec2_float_type = state.builder.AllocateId();
|
||||
state.vec3_float_type = state.builder.AllocateId();
|
||||
state.vec4_float_type = state.builder.AllocateId();
|
||||
state.ptr_func_uint = state.builder.AllocateId();
|
||||
state.ptr_func_bool = state.builder.AllocateId();
|
||||
state.ptr_func_float = state.builder.AllocateId();
|
||||
state.ptr_func_uint_pair = state.builder.AllocateId();
|
||||
state.ptr_func_vec4_uint = state.builder.AllocateId();
|
||||
state.ptr_func_vec2_float = state.builder.AllocateId();
|
||||
state.ptr_input_float = state.builder.AllocateId();
|
||||
state.ptr_input_bool = state.builder.AllocateId();
|
||||
state.ptr_input_int = state.builder.AllocateId();
|
||||
state.ptr_input_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec2_float = state.builder.AllocateId();
|
||||
state.ptr_input_vec3_float = state.builder.AllocateId();
|
||||
state.ptr_input_vec2_int = state.builder.AllocateId();
|
||||
state.ptr_input_vec3_int = state.builder.AllocateId();
|
||||
state.ptr_input_vec4_int = state.builder.AllocateId();
|
||||
state.ptr_input_vec2_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec3_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec4_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec4_float = state.builder.AllocateId();
|
||||
state.sample_mask_array_type = state.builder.AllocateId();
|
||||
state.ptr_output_int = state.builder.AllocateId();
|
||||
state.ptr_output_sample_mask_array = state.builder.AllocateId();
|
||||
state.ptr_output_float = state.builder.AllocateId();
|
||||
state.ptr_output_vec4_float = state.builder.AllocateId();
|
||||
const auto ptr_output_vec4_uint = state.builder.AllocateId();
|
||||
state.per_vertex_type = state.builder.AllocateId();
|
||||
state.ptr_output_per_vertex = state.builder.AllocateId();
|
||||
state.storage_runtime_array_type = state.builder.AllocateId();
|
||||
state.storage_buffer_type = state.builder.AllocateId();
|
||||
state.ptr_storage_buffer = state.builder.AllocateId();
|
||||
state.ptr_storage_buffer_uint = state.builder.AllocateId();
|
||||
state.storage_buffer_array_type = state.builder.AllocateId();
|
||||
state.ptr_storage_buffer_array = state.builder.AllocateId();
|
||||
if (state.address_memory_variable != 0) {
|
||||
state.address_memory_array_type = state.builder.AllocateId();
|
||||
state.ptr_address_memory_array = state.builder.AllocateId();
|
||||
}
|
||||
if (state.push_constant_variable != 0) {
|
||||
state.push_constant_array_type = state.builder.AllocateId();
|
||||
state.push_constant_block_type = state.builder.AllocateId();
|
||||
state.ptr_push_constant_block = state.builder.AllocateId();
|
||||
state.ptr_push_constant_uint = state.builder.AllocateId();
|
||||
}
|
||||
state.lds_array_type = state.builder.AllocateId();
|
||||
state.ptr_workgroup_array = state.builder.AllocateId();
|
||||
state.ptr_workgroup_uint = state.builder.AllocateId();
|
||||
state.lds_variable = state.builder.AllocateId();
|
||||
for (auto& image: state.sampled_images) {
|
||||
image.image_type = state.builder.AllocateId();
|
||||
image.sampled_image_type = state.builder.AllocateId();
|
||||
image.pointer_type = state.builder.AllocateId();
|
||||
image.array_type = state.builder.AllocateId();
|
||||
image.array_pointer_type = state.builder.AllocateId();
|
||||
}
|
||||
for (auto& image: state.storage_images) {
|
||||
image.image_type = state.builder.AllocateId();
|
||||
image.pointer_type = state.builder.AllocateId();
|
||||
image.array_type = state.builder.AllocateId();
|
||||
image.array_pointer_type = state.builder.AllocateId();
|
||||
}
|
||||
state.sampler_type = state.builder.AllocateId();
|
||||
state.sampler_array_type = state.builder.AllocateId();
|
||||
state.ptr_uniform_sampler = state.builder.AllocateId();
|
||||
state.ptr_uniform_sampler_array = state.builder.AllocateId();
|
||||
state.ptr_image_uint = state.builder.AllocateId();
|
||||
state.func_type = state.builder.AllocateId();
|
||||
state.main_func = state.builder.AllocateId();
|
||||
state.entry_label = state.builder.AllocateId();
|
||||
state.glsl_std450 = state.builder.AllocateId();
|
||||
|
||||
state.builder.AddCapability({CapabilityShader});
|
||||
state.builder.AddCapability({CapabilitySampled1D});
|
||||
state.builder.AddCapability({CapabilityImage1D});
|
||||
state.builder.AddCapability({CapabilityImageQuery});
|
||||
if (state.needs_image_gather_extended) {
|
||||
state.builder.AddCapability({CapabilityImageGatherExtended});
|
||||
}
|
||||
if (std::any_of(state.storage_images.begin(),
|
||||
state.storage_images.begin() + StorageImageViewKindCount,
|
||||
[](const auto& image) { return image.variable != 0; })) {
|
||||
state.builder.AddCapability({CapabilityStorageImageReadWithoutFormat});
|
||||
state.builder.AddCapability({CapabilityStorageImageWriteWithoutFormat});
|
||||
}
|
||||
if (state.needs_subgroup_ballot || state.needs_subgroup_shuffle ||
|
||||
state.needs_subgroup_local_invocation_id) {
|
||||
state.builder.AddCapability({CapabilityGroupNonUniform});
|
||||
}
|
||||
if (state.needs_subgroup_ballot) {
|
||||
state.builder.AddCapability({CapabilityGroupNonUniformBallot});
|
||||
}
|
||||
if (state.needs_subgroup_shuffle) {
|
||||
state.builder.AddCapability({CapabilityGroupNonUniformShuffle});
|
||||
}
|
||||
if (state.needs_compute_derivatives && state.stage == ShaderType::Compute) {
|
||||
state.builder.AddCapability({CapabilityComputeDerivativeGroupQuadsKHR});
|
||||
state.builder.AddExtension("SPV_KHR_compute_shader_derivatives");
|
||||
}
|
||||
state.builder.AddExtInstImport(state.glsl_std450, "GLSL.std.450");
|
||||
state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
|
||||
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
|
||||
state.interface_variables);
|
||||
if (state.stage == ShaderType::Compute) {
|
||||
uint32_t local_x = state.needs_compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_y = state.needs_compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_z = 1u;
|
||||
if (state.compute_input_info != nullptr) {
|
||||
const auto* cs = state.compute_input_info;
|
||||
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
|
||||
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
|
||||
local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
|
||||
}
|
||||
state.builder.AddExecutionMode(
|
||||
{state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
|
||||
}
|
||||
if (state.stage == ShaderType::Pixel) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeOriginUpperLeft});
|
||||
if (state.depth_variable != 0) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeDepthReplacing});
|
||||
}
|
||||
if (state.pixel_input_info != nullptr && state.pixel_input_info->ps_early_z &&
|
||||
!state.pixel_input_info->ps_pixel_kill_enable &&
|
||||
!state.pixel_input_info->ps_depth_export_enable &&
|
||||
!state.pixel_input_info->ps_sample_mask_export_enable) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeEarlyFragmentTests});
|
||||
}
|
||||
}
|
||||
if (state.needs_compute_derivatives && state.stage == ShaderType::Compute) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeDerivativeGroupQuadsKHR});
|
||||
}
|
||||
state.builder.AddName(state.main_func, "main");
|
||||
if (state.stage == ShaderType::Compute || state.needs_function_lds) {
|
||||
state.builder.AddName(state.lds_variable, "lds_dwords");
|
||||
}
|
||||
AddInputAnnotationsAndNames(state);
|
||||
AddOutputAnnotationsAndNames(state);
|
||||
AddDescriptorAnnotationsAndNames(state);
|
||||
AddVsharpAnnotationsAndNames(state);
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.storage_runtime_array_type, DecorationArrayStride, 4});
|
||||
state.builder.AddAnnotation(
|
||||
{OpMemberDecorate, state.storage_buffer_type, 0, DecorationOffset, 0});
|
||||
state.builder.AddAnnotation({OpDecorate, state.storage_buffer_type, DecorationBlock});
|
||||
|
||||
state.builder.AddType({OpTypeVoid, state.void_type});
|
||||
state.builder.AddType({OpTypeBool, state.bool_type});
|
||||
state.builder.AddType({OpTypeInt, state.uint_type, 32, 0});
|
||||
state.builder.AddType({OpTypeStruct, state.uint_pair_type, state.uint_type, state.uint_type});
|
||||
state.builder.AddType({OpTypeInt, state.int_type, 32, 1});
|
||||
state.builder.AddType({OpTypeStruct, state.int_pair_type, state.int_type, state.int_type});
|
||||
state.builder.AddType({OpTypeFloat, state.float_type, 32});
|
||||
state.builder.AddType({OpTypeVector, state.vec2_uint_type, state.uint_type, 2});
|
||||
state.builder.AddType({OpTypeVector, state.vec3_uint_type, state.uint_type, 3});
|
||||
state.builder.AddType({OpTypeVector, state.vec4_uint_type, state.uint_type, 4});
|
||||
state.builder.AddType({OpTypeVector, state.vec2_int_type, state.int_type, 2});
|
||||
state.builder.AddType({OpTypeVector, state.vec3_int_type, state.int_type, 3});
|
||||
state.builder.AddType({OpTypeVector, state.vec4_int_type, state.int_type, 4});
|
||||
state.builder.AddType({OpTypeVector, state.vec2_float_type, state.float_type, 2});
|
||||
state.builder.AddType({OpTypeVector, state.vec3_float_type, state.float_type, 3});
|
||||
state.builder.AddType({OpTypeVector, state.vec4_float_type, state.float_type, 4});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_uint, StorageClassFunction, state.uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_bool, StorageClassFunction, state.bool_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_float, StorageClassFunction, state.float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_uint_pair, StorageClassFunction, state.uint_pair_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_vec4_uint, StorageClassFunction, state.vec4_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_vec2_float, StorageClassFunction, state.vec2_float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_float, StorageClassInput, state.float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_bool, StorageClassInput, state.bool_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_input_int, StorageClassInput, state.int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_uint, StorageClassInput, state.uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec2_float, StorageClassInput, state.vec2_float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec3_float, StorageClassInput, state.vec3_float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec2_int, StorageClassInput, state.vec2_int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec3_int, StorageClassInput, state.vec3_int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec4_int, StorageClassInput, state.vec4_int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec2_uint, StorageClassInput, state.vec2_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec3_uint, StorageClassInput, state.vec3_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec4_uint, StorageClassInput, state.vec4_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec4_float, StorageClassInput, state.vec4_float_type});
|
||||
if (state.subgroup_local_invocation_id_variable != 0) {
|
||||
state.builder.AddType({OpVariable, state.ptr_input_uint,
|
||||
state.subgroup_local_invocation_id_variable, StorageClassInput});
|
||||
}
|
||||
for (const auto& input: state.inputs) {
|
||||
uint32_t ptr_type = state.ptr_input_uint;
|
||||
switch (input.kind) {
|
||||
case IR::StageInputKind::VertexIndex:
|
||||
case IR::StageInputKind::InstanceIndex: ptr_type = state.ptr_input_int; break;
|
||||
case IR::StageInputKind::WorkgroupId:
|
||||
case IR::StageInputKind::LocalInvocationId:
|
||||
case IR::StageInputKind::GlobalInvocationId:
|
||||
ptr_type = state.ptr_input_vec3_uint;
|
||||
break;
|
||||
case IR::StageInputKind::FragCoord: ptr_type = state.ptr_input_vec4_float; break;
|
||||
case IR::StageInputKind::FrontFacing: ptr_type = state.ptr_input_bool; break;
|
||||
case IR::StageInputKind::Parameter:
|
||||
if (state.stage == ShaderType::Vertex) {
|
||||
const auto kind = VertexParameterScalarKind(state, input.location);
|
||||
const auto components = VertexParameterComponentCount(state, input);
|
||||
ptr_type = VertexParameterInputPointerType(state, kind, components);
|
||||
} else {
|
||||
ptr_type = state.ptr_input_vec4_float;
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
state.builder.AddType({OpVariable, ptr_type, input.variable_id, StorageClassInput});
|
||||
}
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_output_float, StorageClassOutput, state.float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_output_int, StorageClassOutput, state.int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_output_vec4_float, StorageClassOutput, state.vec4_float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, ptr_output_vec4_uint, StorageClassOutput, state.vec4_uint_type});
|
||||
if (state.per_vertex_variable != 0) {
|
||||
state.builder.AddType({OpTypeStruct, state.per_vertex_type, state.vec4_float_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_output_per_vertex, StorageClassOutput,
|
||||
state.per_vertex_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_output_per_vertex, state.per_vertex_variable,
|
||||
StorageClassOutput});
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
if (binding.kind == IR::StageOutputKind::Parameter ||
|
||||
binding.kind == IR::StageOutputKind::Mrt) {
|
||||
const auto pointer_type =
|
||||
binding.kind == IR::StageOutputKind::Mrt && MrtUsesUintOutput(state, binding.index)
|
||||
? ptr_output_vec4_uint
|
||||
: state.ptr_output_vec4_float;
|
||||
state.builder.AddType(
|
||||
{OpVariable, pointer_type, binding.variable_id, StorageClassOutput});
|
||||
}
|
||||
}
|
||||
if (state.depth_variable != 0) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_output_float, state.depth_variable, StorageClassOutput});
|
||||
}
|
||||
if (state.sample_mask_variable != 0) {
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.sample_mask_array_type, state.int_type, ConstantU32(state, 1)});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_output_sample_mask_array,
|
||||
StorageClassOutput, state.sample_mask_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_output_sample_mask_array,
|
||||
state.sample_mask_variable, StorageClassOutput});
|
||||
}
|
||||
state.builder.AddType({OpTypeRuntimeArray, state.storage_runtime_array_type, state.uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypeStruct, state.storage_buffer_type, state.storage_runtime_array_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_storage_buffer, StorageClassStorageBuffer,
|
||||
state.storage_buffer_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_storage_buffer_uint, StorageClassStorageBuffer, state.uint_type});
|
||||
if (state.storage_buffer_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Buffers));
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.storage_buffer_array_type, state.storage_buffer_type, count});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_storage_buffer_array,
|
||||
StorageClassStorageBuffer, state.storage_buffer_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_storage_buffer_array,
|
||||
state.storage_buffer_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.gds_variable != 0) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_storage_buffer, state.gds_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.push_constant_variable != 0) {
|
||||
const auto dword_count =
|
||||
ConstantU32(state, state.program.bindings.push_constant_size / sizeof(uint32_t));
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.push_constant_array_type, state.uint_type, dword_count});
|
||||
state.builder.AddType(
|
||||
{OpTypeStruct, state.push_constant_block_type, state.push_constant_array_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_push_constant_block,
|
||||
StorageClassPushConstant, state.push_constant_block_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_push_constant_uint,
|
||||
StorageClassPushConstant, state.uint_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_push_constant_block,
|
||||
state.push_constant_variable, StorageClassPushConstant});
|
||||
}
|
||||
if (state.vsharp_storage_variable != 0) {
|
||||
state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.vsharp_storage_variable,
|
||||
StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.address_memory_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::AddressMemory));
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.address_memory_array_type, state.storage_buffer_type, count});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_address_memory_array,
|
||||
StorageClassStorageBuffer, state.address_memory_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_address_memory_array,
|
||||
state.address_memory_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.flattened_srt_variable != 0) {
|
||||
state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.flattened_srt_variable,
|
||||
StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.stage == ShaderType::Compute || state.needs_function_lds) {
|
||||
const auto storage_class =
|
||||
state.needs_function_lds ? StorageClassFunction : StorageClassWorkgroup;
|
||||
const auto lds_size = ConstantU32(state, state.needs_function_lds ? 8192u : 1024u);
|
||||
state.builder.AddType({OpTypeArray, state.lds_array_type, state.uint_type, lds_size});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_workgroup_array, storage_class, state.lds_array_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_workgroup_uint, storage_class, state.uint_type});
|
||||
if (state.stage == ShaderType::Compute) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassWorkgroup});
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < state.sampled_images.size(); i++) {
|
||||
auto& image = state.sampled_images[i];
|
||||
const auto view = static_cast<ImageViewKind>(i % SampledImageViewKindCount);
|
||||
const bool integer = i >= SampledImageViewKindCount;
|
||||
const auto component = integer ? state.uint_type : state.float_type;
|
||||
state.builder.AddType({OpTypeImage, image.image_type, component, ImageSpirvDimension(view),
|
||||
0, ImageSpirvArrayed(view), ImageSpirvMultisampled(view), 1,
|
||||
ImageFormatUnknown});
|
||||
state.builder.AddType({OpTypeSampledImage, image.sampled_image_type, image.image_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type});
|
||||
if (image.variable != 0) {
|
||||
const auto kind = SampledBindingKind(integer, view);
|
||||
const auto count = ConstantU32(state, DescriptorCount(state, kind));
|
||||
state.builder.AddType({OpTypeArray, image.array_type, image.image_type, count});
|
||||
state.builder.AddType({OpTypePointer, image.array_pointer_type,
|
||||
StorageClassUniformConstant, image.array_type});
|
||||
state.builder.AddType({OpVariable, image.array_pointer_type, image.variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
}
|
||||
state.builder.AddType({OpTypeSampler, state.sampler_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_uniform_sampler, StorageClassUniformConstant,
|
||||
state.sampler_type});
|
||||
if (state.sampler_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Samplers));
|
||||
state.builder.AddType({OpTypeArray, state.sampler_array_type, state.sampler_type, count});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_uniform_sampler_array,
|
||||
StorageClassUniformConstant, state.sampler_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_uniform_sampler_array, state.sampler_variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
for (uint32_t i = 0; i < state.storage_images.size(); i++) {
|
||||
auto& image = state.storage_images[i];
|
||||
const auto view = static_cast<ImageViewKind>(i % StorageImageViewKindCount);
|
||||
const bool integer = i >= StorageImageViewKindCount;
|
||||
const auto component = integer ? state.uint_type : state.float_type;
|
||||
const auto format = integer ? ImageFormatR32ui : ImageFormatUnknown;
|
||||
state.builder.AddType({OpTypeImage, image.image_type, component, ImageSpirvDimension(view),
|
||||
0, ImageSpirvArrayed(view), 0, 2, format});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type});
|
||||
if (image.variable != 0) {
|
||||
const auto kind = StorageBindingKind(integer, view);
|
||||
const auto count = ConstantU32(state, DescriptorCount(state, kind));
|
||||
state.builder.AddType({OpTypeArray, image.array_type, image.image_type, count});
|
||||
state.builder.AddType({OpTypePointer, image.array_pointer_type,
|
||||
StorageClassUniformConstant, image.array_type});
|
||||
state.builder.AddType({OpVariable, image.array_pointer_type, image.variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
}
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_image_uint, StorageClassImage, state.uint_type});
|
||||
state.builder.AddType({OpTypeFunction, state.func_type, state.void_type});
|
||||
}
|
||||
|
||||
void AllocateRegisterVariables(EmitterState& state) {
|
||||
if (state.stage == ShaderType::Pixel && state.needs_pixel_valid_mask) {
|
||||
state.pixel_valid_mask_variable = state.builder.AllocateId();
|
||||
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
|
||||
}
|
||||
for (auto& binding: state.registers) {
|
||||
binding.pointer_id = state.builder.AllocateId();
|
||||
const auto name = IR::RegisterToString(binding.reg);
|
||||
state.builder.AddName(binding.pointer_id, name.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
void AllocateDescriptorVariables(EmitterState& state) {
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::Buffers) != nullptr) {
|
||||
state.storage_buffer_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::AddressMemory) != nullptr) {
|
||||
state.address_memory_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (state.program.bindings.push_constant_size != 0) {
|
||||
state.push_constant_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::UserData) != nullptr) {
|
||||
state.vsharp_storage_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
|
||||
state.flattened_srt_variable = state.builder.AllocateId();
|
||||
}
|
||||
for (uint32_t i = 0; i < state.sampled_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % SampledImageViewKindCount);
|
||||
if (DescriptorBinding(state, SampledBindingKind(i >= SampledImageViewKindCount, view)) !=
|
||||
nullptr) {
|
||||
state.sampled_images[i].variable = state.builder.AllocateId();
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < state.storage_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % StorageImageViewKindCount);
|
||||
if (DescriptorBinding(state, StorageBindingKind(i >= StorageImageViewKindCount, view)) !=
|
||||
nullptr) {
|
||||
state.storage_images[i].variable = state.builder.AllocateId();
|
||||
}
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::Samplers) != nullptr) {
|
||||
state.sampler_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::Gds) != nullptr) {
|
||||
state.gds_variable = state.builder.AllocateId();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,541 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <fmt/format.h>
|
||||
#include <functional>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t BoolConstant(EmitterState& state, bool value) {
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.builder.AddType({value ? 41u : 42u, state.bool_type, id});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantPair(EmitterState& state, uint64_t value) {
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, state.uint_pair_type, id,
|
||||
ConstantU32(state, static_cast<uint32_t>(value)),
|
||||
ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t PhiPointerType(const EmitterState& state, IR::Type type) {
|
||||
switch (type) {
|
||||
case IR::Type::U1: return state.ptr_func_bool;
|
||||
case IR::Type::U8:
|
||||
case IR::Type::U16:
|
||||
case IR::Type::U32:
|
||||
case IR::Type::F16: return state.ptr_func_uint;
|
||||
case IR::Type::F32: return state.ptr_func_float;
|
||||
case IR::Type::U64:
|
||||
case IR::Type::U32x2: return state.ptr_func_uint_pair;
|
||||
case IR::Type::U32x4: return state.ptr_func_vec4_uint;
|
||||
case IR::Type::F32x2: return state.ptr_func_vec2_float;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void StorePhiEdge(ValueEmitContext& ctx, const IR::Block* from, const IR::Block* to) {
|
||||
if (to == nullptr) {
|
||||
return;
|
||||
}
|
||||
for (const auto& phi: *to) {
|
||||
if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
|
||||
continue;
|
||||
}
|
||||
for (size_t index = 0; index < phi.NumArgs(); index++) {
|
||||
if (phi.PhiBlock(index) == from) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpStore, ctx.phi_variables.at(&phi), ctx.Def(phi.Arg(index))});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) {
|
||||
const auto found = std::ranges::find_if(
|
||||
program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; });
|
||||
if (found == program.block_info.end()) {
|
||||
return nullptr;
|
||||
}
|
||||
return program.blocks[static_cast<size_t>(found - program.block_info.begin())];
|
||||
}
|
||||
|
||||
void EmitReturn(ValueEmitContext& ctx) {
|
||||
EmitKillIfPixelValidMaskInactive(ctx.state);
|
||||
ctx.state.builder.AddFunction({OpReturn});
|
||||
}
|
||||
|
||||
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
|
||||
const IR::ValueBlockInfo& info) {
|
||||
const auto& term = info.terminator;
|
||||
const auto emit_merge = [&]() {
|
||||
if (term.loop_header) {
|
||||
const auto* merge = TargetBlock(ctx.program, term.merge_block);
|
||||
const auto* cont = TargetBlock(ctx.program, term.continue_block);
|
||||
if (merge != nullptr && cont != nullptr) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone});
|
||||
}
|
||||
} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
|
||||
term.merge_block != UINT32_MAX) {
|
||||
if (const auto* merge = TargetBlock(ctx.program, term.merge_block); merge != nullptr) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpSelectionMerge, ctx.Label(merge), SelectionControlNone});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
switch (term.kind) {
|
||||
case CFG::TerminatorKind::Branch: {
|
||||
const auto* target = TargetBlock(ctx.program, term.true_block);
|
||||
if (target == nullptr) {
|
||||
EmitReturn(ctx);
|
||||
return;
|
||||
}
|
||||
StorePhiEdge(ctx, block, target);
|
||||
emit_merge();
|
||||
ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
|
||||
return;
|
||||
}
|
||||
case CFG::TerminatorKind::ConditionalBranch: {
|
||||
const auto* true_block = TargetBlock(ctx.program, term.true_block);
|
||||
const auto* false_block = TargetBlock(ctx.program, term.false_block);
|
||||
if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) {
|
||||
EmitReturn(ctx);
|
||||
return;
|
||||
}
|
||||
StorePhiEdge(ctx, block, true_block);
|
||||
StorePhiEdge(ctx, block, false_block);
|
||||
const auto condition = ctx.Def(info.condition);
|
||||
emit_merge();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
|
||||
return;
|
||||
}
|
||||
default: EmitReturn(ctx); return;
|
||||
}
|
||||
}
|
||||
|
||||
void EmitDispatcherTarget(ValueEmitContext& ctx, const IR::Block* from, uint32_t target) {
|
||||
const auto* block = TargetBlock(ctx.program, target);
|
||||
if (block != nullptr) {
|
||||
StorePhiEdge(ctx, from, block);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
|
||||
const IR::ValueBlockInfo& info, uint32_t exit_label) {
|
||||
const auto& term = info.terminator;
|
||||
switch (term.kind) {
|
||||
case CFG::TerminatorKind::Branch:
|
||||
EmitDispatcherTarget(ctx, block, term.true_block);
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, term.true_block)});
|
||||
break;
|
||||
case CFG::TerminatorKind::ConditionalBranch: {
|
||||
EmitDispatcherTarget(ctx, block, term.true_block);
|
||||
EmitDispatcherTarget(ctx, block, term.false_block);
|
||||
const auto selected = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, selected,
|
||||
ctx.Def(info.condition),
|
||||
ConstantU32(ctx.state, term.true_block),
|
||||
ConstantU32(ctx.state, term.false_block)});
|
||||
ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
|
||||
break;
|
||||
}
|
||||
case CFG::TerminatorKind::IndirectBranch: {
|
||||
for (const auto target: term.indirect_targets) {
|
||||
EmitDispatcherTarget(ctx, block, target);
|
||||
}
|
||||
for (const auto target: term.indirect_selector_targets) {
|
||||
EmitDispatcherTarget(ctx, block, target);
|
||||
}
|
||||
uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
|
||||
if (!info.indirect_target.IsEmpty()) {
|
||||
const auto selector = ctx.Def(info.indirect_target);
|
||||
const auto& values = term.indirect_selector_code != UINT32_MAX
|
||||
? term.indirect_selector_values
|
||||
: term.indirect_target_pcs;
|
||||
const auto& targets = term.indirect_selector_code != UINT32_MAX
|
||||
? term.indirect_selector_targets
|
||||
: term.indirect_targets;
|
||||
for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
|
||||
const auto match = ctx.state.builder.AllocateId();
|
||||
const auto next = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpIEqual, ctx.state.bool_type, match, selector,
|
||||
ConstantU32(ctx.state, values[index])});
|
||||
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, next, match,
|
||||
ConstantU32(ctx.state, targets[index]),
|
||||
selected});
|
||||
selected = next;
|
||||
}
|
||||
}
|
||||
ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
|
||||
break;
|
||||
}
|
||||
ctx.state.builder.AddFunction({OpBranch, exit_label});
|
||||
}
|
||||
|
||||
bool EmitInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
const auto finish = [&]() {
|
||||
if (const auto found = ctx.dispatcher_variables.find(&inst);
|
||||
found != ctx.dispatcher_variables.end()) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
|
||||
}
|
||||
return true;
|
||||
};
|
||||
if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpLoad, ctx.TypeId(inst.GetType()), ctx.Result(inst), ctx.phi_variables.at(&inst)});
|
||||
return finish();
|
||||
}
|
||||
if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
|
||||
EmitValueImage(ctx, inst)) {
|
||||
return finish();
|
||||
}
|
||||
ctx.Fail(inst, "has no direct SPIR-V emitter");
|
||||
return false;
|
||||
}
|
||||
|
||||
void EmitBlock(ValueEmitContext& ctx, const IR::Block* block) {
|
||||
ctx.current_block = block;
|
||||
ctx.state.builder.AddFunction({OpLabel, ctx.Label(block)});
|
||||
for (const auto& inst: *block) {
|
||||
if (!EmitInstruction(ctx, inst)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void EmitStructuredFunction(ValueEmitContext& ctx) {
|
||||
ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
|
||||
for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
|
||||
EmitBlock(ctx, ctx.program.blocks[index]);
|
||||
EmitStructuredTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index]);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitDispatcherFunction(ValueEmitContext& ctx) {
|
||||
auto& state = ctx.state;
|
||||
const auto* entry = ctx.program.blocks.front();
|
||||
state.builder.AddFunction({OpBranch, ctx.Label(entry)});
|
||||
EmitBlock(ctx, entry);
|
||||
EmitDispatcherTerminator(ctx, entry, ctx.program.block_info.front(),
|
||||
state.dispatch_header_label);
|
||||
|
||||
state.builder.AddFunction({OpLabel, state.dispatch_header_label});
|
||||
const auto pc = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, pc, state.dispatch_pc_variable});
|
||||
const auto done = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, done, pc, ConstantU32(ctx.state, UINT32_MAX)});
|
||||
state.builder.AddFunction(
|
||||
{OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone});
|
||||
state.builder.AddFunction(
|
||||
{OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
|
||||
|
||||
state.builder.AddFunction({OpLabel, state.dispatch_select_label});
|
||||
state.builder.AddFunction(
|
||||
{OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
|
||||
std::vector<uint32_t> words {OpSwitch, pc, state.dispatch_default_label};
|
||||
for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
|
||||
words.push_back(ctx.program.block_info[index].id);
|
||||
words.push_back(ctx.Label(ctx.program.blocks[index]));
|
||||
}
|
||||
state.builder.AddFunction(words);
|
||||
state.builder.AddFunction({OpLabel, state.dispatch_default_label});
|
||||
state.builder.AddFunction(
|
||||
{OpStore, state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
|
||||
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
|
||||
|
||||
for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) {
|
||||
EmitBlock(ctx, ctx.program.blocks[index]);
|
||||
EmitDispatcherTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index],
|
||||
state.dispatch_after_switch_label);
|
||||
}
|
||||
state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
|
||||
state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
|
||||
state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
|
||||
state.builder.AddFunction({OpBranch, state.dispatch_header_label});
|
||||
state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
|
||||
EmitReturn(ctx);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
uint32_t ValueEmitContext::TypeId(IR::Type type) const {
|
||||
switch (type) {
|
||||
case IR::Type::U1: return state.bool_type;
|
||||
case IR::Type::U8:
|
||||
case IR::Type::U16:
|
||||
case IR::Type::U32:
|
||||
case IR::Type::F16: return state.uint_type;
|
||||
case IR::Type::U64:
|
||||
case IR::Type::U32x2: return state.uint_pair_type;
|
||||
case IR::Type::F32: return state.float_type;
|
||||
case IR::Type::U32x4: return state.vec4_uint_type;
|
||||
case IR::Type::F32x2: return state.vec2_float_type;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::Def(IR::Value value) {
|
||||
value = value.Resolve();
|
||||
if (value.IsImmediate()) {
|
||||
switch (value.GetType()) {
|
||||
case IR::Type::U1: return BoolConstant(state, value.U1());
|
||||
case IR::Type::U8: return ConstantU32(state, value.U8());
|
||||
case IR::Type::U16: return ConstantU32(state, value.U16());
|
||||
case IR::Type::U32: return ConstantU32(state, value.U32());
|
||||
case IR::Type::U64: return ConstantPair(state, value.U64());
|
||||
case IR::Type::F16: return ConstantU32(state, value.F16Bits());
|
||||
case IR::Type::F32:
|
||||
return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
const auto* inst = value.ResolveInstruction();
|
||||
if (inst == nullptr) {
|
||||
failed = true;
|
||||
error = "direct SPIR-V emitter received a non-value argument";
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
if (program.dispatcher_fallback && current_block != nullptr &&
|
||||
inst->Parent() != current_block) {
|
||||
if (const auto found = dispatcher_variables.find(inst);
|
||||
found != dispatcher_variables.end()) {
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
|
||||
return id;
|
||||
}
|
||||
}
|
||||
if (const auto found = definitions.find(inst); found != definitions.end()) {
|
||||
return found->second;
|
||||
}
|
||||
const auto id = state.builder.AllocateId();
|
||||
definitions.emplace(inst, id);
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
|
||||
return Def(inst.Arg(index));
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
|
||||
if (const auto found = definitions.find(&inst); found != definitions.end()) {
|
||||
return found->second;
|
||||
}
|
||||
const auto id = state.builder.AllocateId();
|
||||
definitions.emplace(&inst, id);
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
|
||||
std::initializer_list<uint32_t> args) {
|
||||
std::vector<uint32_t> words {opcode, TypeId(type), Result(inst)};
|
||||
words.insert(words.end(), args.begin(), args.end());
|
||||
state.builder.AddFunction(words);
|
||||
return Result(inst);
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
|
||||
if (const auto found = definitions.find(&inst); found != definitions.end()) {
|
||||
if (found->second != value) {
|
||||
state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value});
|
||||
}
|
||||
return found->second;
|
||||
}
|
||||
definitions.emplace(&inst, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) {
|
||||
const auto* inst = value.ResolveInstruction();
|
||||
if (inst == nullptr || inst->GetOpcode() != opcode) {
|
||||
failed = true;
|
||||
error = "typed resource handle has the wrong producer";
|
||||
return 0;
|
||||
}
|
||||
return inst->Flags<uint32_t>();
|
||||
}
|
||||
|
||||
const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) {
|
||||
const auto* inst = value.ResolveInstruction();
|
||||
if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) {
|
||||
failed = true;
|
||||
error = "typed image address was not constructed by MakeImageAddress";
|
||||
return nullptr;
|
||||
}
|
||||
return inst;
|
||||
}
|
||||
|
||||
const IR::MemoryInfo& ValueEmitContext::Memory(const IR::Inst& inst) const {
|
||||
return program.memory_info.at(inst.Flags<IR::MemoryFlags>().index);
|
||||
}
|
||||
|
||||
const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const {
|
||||
return program.export_info.at(inst.Flags<IR::ExportFlags>().index);
|
||||
}
|
||||
|
||||
uint32_t ValueEmitContext::Label(const IR::Block* block) const {
|
||||
return labels.at(block);
|
||||
}
|
||||
|
||||
size_t ValueEmitContext::BlockIndex(const IR::Block* block) const {
|
||||
return static_cast<size_t>(std::ranges::find(program.blocks, block) - program.blocks.begin());
|
||||
}
|
||||
|
||||
void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
|
||||
failed = true;
|
||||
error = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
|
||||
}
|
||||
|
||||
bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) {
|
||||
ValueEmitContext ctx(state, program);
|
||||
for (const auto* block: program.blocks) {
|
||||
ctx.labels.emplace(block, state.builder.AllocateId());
|
||||
for (const auto& inst: *block) {
|
||||
if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
|
||||
continue;
|
||||
}
|
||||
const auto ptr_type = PhiPointerType(state, inst.GetType());
|
||||
if (ptr_type == 0) {
|
||||
ctx.failed = true;
|
||||
ctx.error = fmt::format("typed opcode Phi of type {} cannot be spilled across a "
|
||||
"control-flow edge",
|
||||
IR::TypeName(inst.GetType()));
|
||||
break;
|
||||
}
|
||||
ctx.phi_variables.emplace(&inst, state.builder.AllocateId());
|
||||
}
|
||||
}
|
||||
if (ctx.failed) {
|
||||
SetError(error, ctx.error.c_str());
|
||||
return false;
|
||||
}
|
||||
if (program.dispatcher_fallback) {
|
||||
const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
|
||||
value = value.Resolve();
|
||||
const auto* definition = value.TryInstruction();
|
||||
if (definition == nullptr || definition->Parent() == consumer ||
|
||||
PhiPointerType(state, definition->GetType()) == 0) {
|
||||
return;
|
||||
}
|
||||
if (!ctx.dispatcher_variables.contains(definition)) {
|
||||
ctx.dispatcher_variables.emplace(definition, state.builder.AllocateId());
|
||||
}
|
||||
};
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
for (size_t index = 0; index < inst.NumArgs(); index++) {
|
||||
const auto* consumer =
|
||||
inst.GetOpcode() == IR::ValueOpcode::Phi ? inst.PhiBlock(index) : block;
|
||||
mark_cross_block(inst.Arg(index), consumer);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t index = 0; index < program.blocks.size(); index++) {
|
||||
mark_cross_block(program.block_info[index].condition, program.blocks[index]);
|
||||
mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
|
||||
}
|
||||
state.dispatcher_fallback = true;
|
||||
state.dispatch_pc_variable = state.builder.AllocateId();
|
||||
state.dispatch_header_label = state.builder.AllocateId();
|
||||
state.dispatch_select_label = state.builder.AllocateId();
|
||||
state.dispatch_default_label = state.builder.AllocateId();
|
||||
state.dispatch_after_switch_label = state.builder.AllocateId();
|
||||
state.dispatch_continue_label = state.builder.AllocateId();
|
||||
state.dispatch_merge_label = state.builder.AllocateId();
|
||||
}
|
||||
for (const auto* block: program.blocks) {
|
||||
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
|
||||
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
|
||||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
|
||||
inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
|
||||
inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
|
||||
inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
|
||||
})) {
|
||||
ctx.scratch_u32_variable = state.builder.AllocateId();
|
||||
break;
|
||||
}
|
||||
}
|
||||
state.builder.AddFunction(
|
||||
{OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
|
||||
state.builder.AddFunction({OpLabel, state.entry_label});
|
||||
if (state.needs_function_lds) {
|
||||
state.builder.AddFunction(
|
||||
{OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
|
||||
}
|
||||
if (state.pixel_valid_mask_variable != 0) {
|
||||
state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable,
|
||||
StorageClassFunction});
|
||||
}
|
||||
if (program.dispatcher_fallback) {
|
||||
state.builder.AddFunction(
|
||||
{OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
|
||||
}
|
||||
for (const auto& binding: state.registers) {
|
||||
state.builder.AddFunction(
|
||||
{OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
|
||||
}
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
|
||||
continue;
|
||||
}
|
||||
state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
|
||||
ctx.phi_variables.at(&inst), StorageClassFunction});
|
||||
}
|
||||
}
|
||||
if (program.dispatcher_fallback) {
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (const auto found = ctx.dispatcher_variables.find(&inst);
|
||||
found != ctx.dispatcher_variables.end()) {
|
||||
state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
|
||||
found->second, StorageClassFunction});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ctx.scratch_u32_variable != 0) {
|
||||
state.builder.AddFunction(
|
||||
{OpVariable, state.ptr_func_uint, ctx.scratch_u32_variable, StorageClassFunction});
|
||||
}
|
||||
for (const auto& binding: state.registers) {
|
||||
state.builder.AddFunction({OpStore, binding.pointer_id,
|
||||
ConstantU32(state, InitialRegisterValue(state, binding.reg))});
|
||||
}
|
||||
if (state.pixel_valid_mask_variable != 0) {
|
||||
state.builder.AddFunction(
|
||||
{OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
|
||||
}
|
||||
EmitStorageBufferOffsets(state);
|
||||
if (program.blocks.empty()) {
|
||||
EmitReturn(ctx);
|
||||
} else if (program.dispatcher_fallback) {
|
||||
EmitDispatcherFunction(ctx);
|
||||
} else {
|
||||
EmitStructuredFunction(ctx);
|
||||
}
|
||||
state.builder.AddFunction({OpFunctionEnd});
|
||||
if (ctx.failed) {
|
||||
SetError(error, ctx.error.c_str());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,677 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <array>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
struct Pair {
|
||||
uint32_t low = 0;
|
||||
uint32_t high = 0;
|
||||
};
|
||||
|
||||
uint32_t NewUnary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, value});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t NewBinary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
|
||||
uint32_t rhs) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, lhs, rhs});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
|
||||
uint32_t false_value) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
|
||||
return result;
|
||||
}
|
||||
|
||||
Pair ExtractPair(EmitterState& state, uint32_t value) {
|
||||
Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.low, value, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.high, value, 1});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.uint_pair_type, result, low, high});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
|
||||
uint32_t high_compare, uint32_t low_compare, bool equal_compare = false,
|
||||
bool not_equal = false) {
|
||||
const auto lhs = ExtractPair(state, lhs_value);
|
||||
const auto rhs = ExtractPair(state, rhs_value);
|
||||
const auto high_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.high, rhs.high);
|
||||
const auto low_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.low, rhs.low);
|
||||
if (equal_compare) {
|
||||
return NewBinary(
|
||||
state, not_equal ? OpLogicalOr : OpLogicalAnd, state.bool_type,
|
||||
not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, high_equal) : high_equal,
|
||||
not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, low_equal) : low_equal);
|
||||
}
|
||||
const auto high_result = NewBinary(state, high_compare, state.bool_type, lhs.high, rhs.high);
|
||||
const auto low_result = NewBinary(state, low_compare, state.bool_type, lhs.low, rhs.low);
|
||||
const auto low_path = NewBinary(state, OpLogicalAnd, state.bool_type, high_equal, low_result);
|
||||
return NewBinary(state, OpLogicalOr, state.bool_type, high_result, low_path);
|
||||
}
|
||||
|
||||
uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
|
||||
const auto lhs = ExtractPair(state, lhs_value);
|
||||
const auto rhs = ExtractPair(state, rhs_value);
|
||||
const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0));
|
||||
const auto high0 = NewBinary(state, OpIAdd, state.uint_type, lhs.high, rhs.high);
|
||||
const auto high = NewBinary(state, OpIAdd, state.uint_type, high0, carry.carry);
|
||||
return MakePair(state, carry.sum, high);
|
||||
}
|
||||
|
||||
uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
|
||||
const auto lhs = ExtractPair(state, lhs_value);
|
||||
const auto rhs = ExtractPair(state, rhs_value);
|
||||
const auto low = NewBinary(state, OpISub, state.uint_type, lhs.low, rhs.low);
|
||||
const auto borrow = NewBinary(state, OpULessThan, state.bool_type, lhs.low, rhs.low);
|
||||
const auto borrow_u32 =
|
||||
NewSelect(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0));
|
||||
const auto high0 = NewBinary(state, OpISub, state.uint_type, lhs.high, rhs.high);
|
||||
const auto high = NewBinary(state, OpISub, state.uint_type, high0, borrow_u32);
|
||||
return MakePair(state, low, high);
|
||||
}
|
||||
|
||||
uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
|
||||
const auto operand_type = signed_value ? state.int_type : state.uint_type;
|
||||
const auto pair_type = signed_value ? state.int_pair_type : state.uint_pair_type;
|
||||
uint32_t lhs_operand = lhs;
|
||||
uint32_t rhs_operand = rhs;
|
||||
if (signed_value) {
|
||||
lhs_operand = NewUnary(state, OpBitcast, state.int_type, lhs);
|
||||
rhs_operand = NewUnary(state, OpBitcast, state.int_type, rhs);
|
||||
}
|
||||
const auto extended = state.builder.AllocateId();
|
||||
state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended,
|
||||
lhs_operand, rhs_operand});
|
||||
const auto high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1});
|
||||
return signed_value ? NewUnary(state, OpBitcast, state.uint_type, high) : high;
|
||||
}
|
||||
|
||||
uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
|
||||
const auto lhs = ExtractPair(state, lhs_value);
|
||||
const auto rhs = ExtractPair(state, rhs_value);
|
||||
const auto low = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.low);
|
||||
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
|
||||
const auto high1 = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.high);
|
||||
const auto high2 = NewBinary(state, OpIMul, state.uint_type, lhs.high, rhs.low);
|
||||
return MakePair(state, low,
|
||||
NewBinary(state, OpIAdd, state.uint_type,
|
||||
NewBinary(state, OpIAdd, state.uint_type, high0, high1), high2));
|
||||
}
|
||||
|
||||
uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value,
|
||||
uint32_t rhs_value) {
|
||||
const auto lhs = ExtractPair(state, lhs_value);
|
||||
const auto rhs = ExtractPair(state, rhs_value);
|
||||
return MakePair(state, NewBinary(state, opcode, state.uint_type, lhs.low, rhs.low),
|
||||
NewBinary(state, opcode, state.uint_type, lhs.high, rhs.high));
|
||||
}
|
||||
|
||||
uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
|
||||
const auto pair = ExtractPair(state, value);
|
||||
uint32_t low = 0;
|
||||
uint32_t high = 0;
|
||||
if (opcode == OpShiftLeftLogical) {
|
||||
EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high);
|
||||
} else if (opcode == OpShiftRightLogical) {
|
||||
EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
|
||||
} else {
|
||||
const auto logical = EmitShift64(state, OpShiftRightLogical, value, shift);
|
||||
const auto logical_pair = ExtractPair(state, logical);
|
||||
const auto sign =
|
||||
NewBinary(state, OpSLessThan, state.bool_type, pair.high, ConstantU32(state, 0));
|
||||
const auto inverse =
|
||||
NewBinary(state, OpISub, state.uint_type, ConstantU32(state, 64), shift);
|
||||
const auto fill = NewBinary(state, OpShiftLeftLogical, state.uint_type,
|
||||
ConstantU32(state, 0xffffffffu), inverse);
|
||||
const auto high_shift =
|
||||
NewBinary(state, OpISub, state.uint_type, shift, ConstantU32(state, 32));
|
||||
const auto high_fill = NewSelect(
|
||||
state, state.uint_type,
|
||||
NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, ConstantU32(state, 32)),
|
||||
ConstantU32(state, 0xffffffffu), fill);
|
||||
const auto fill_low =
|
||||
NewSelect(state, state.uint_type, sign,
|
||||
NewSelect(state, state.uint_type,
|
||||
NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift,
|
||||
ConstantU32(state, 32)),
|
||||
NewBinary(state, OpShiftLeftLogical, state.uint_type,
|
||||
ConstantU32(state, 0xffffffffu), high_shift),
|
||||
ConstantU32(state, 0)),
|
||||
ConstantU32(state, 0));
|
||||
const auto fill_high =
|
||||
NewSelect(state, state.uint_type, sign, high_fill, ConstantU32(state, 0));
|
||||
return MakePair(
|
||||
state, NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.low, fill_low),
|
||||
NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.high, fill_high));
|
||||
}
|
||||
return MakePair(state, low, high);
|
||||
}
|
||||
|
||||
uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
|
||||
const auto pair = ExtractPair(state, value);
|
||||
const auto high_i = state.builder.AllocateId();
|
||||
const auto low_i = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.int_type, high_i, state.glsl_std450, GlslFindUMsb, pair.high});
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.int_type, low_i, state.glsl_std450, GlslFindUMsb, pair.low});
|
||||
const auto high = NewUnary(state, OpBitcast, state.uint_type, high_i);
|
||||
const auto low = NewUnary(state, OpBitcast, state.uint_type, low_i);
|
||||
const auto high_nonzero =
|
||||
NewBinary(state, OpINotEqual, state.bool_type, pair.high, ConstantU32(state, 0));
|
||||
return NewSelect(state, state.uint_type, high_nonzero,
|
||||
NewBinary(state, OpIAdd, state.uint_type, high, ConstantU32(state, 32)), low);
|
||||
}
|
||||
|
||||
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
|
||||
bool max_value) {
|
||||
const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
|
||||
: EmitMinMaxU32Value(state, a, b, max_value);
|
||||
return signed_value ? EmitMinMaxI32Value(state, ab, c, max_value)
|
||||
: EmitMinMaxU32Value(state, ab, c, max_value);
|
||||
}
|
||||
|
||||
uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
|
||||
const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
|
||||
const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
|
||||
const auto ab = NewBinary(state, OpIAdd, state.uint_type, a, b);
|
||||
const auto abc = NewBinary(state, OpIAdd, state.uint_type, ab, c);
|
||||
return NewBinary(state, OpISub, state.uint_type,
|
||||
NewBinary(state, OpISub, state.uint_type, abc, minimum), maximum);
|
||||
}
|
||||
|
||||
uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
|
||||
return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
|
||||
}
|
||||
|
||||
uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
|
||||
const auto min_ab = EmitMinMaxF32Value(state, a, b, false);
|
||||
const auto min3 = EmitMinMaxF32Value(state, min_ab, c, false);
|
||||
const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
|
||||
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
|
||||
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
|
||||
const auto nan_ab = NewBinary(state, OpLogicalOr, state.bool_type,
|
||||
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
|
||||
const auto any_nan =
|
||||
NewBinary(state, OpLogicalOr, state.bool_type, nan_ab, EmitClassifyF32(state, c).nan);
|
||||
return NewSelect(state, state.float_type, any_nan, min3, median);
|
||||
}
|
||||
|
||||
uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
|
||||
std::initializer_list<uint32_t> args) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
std::vector<uint32_t> words {OpExtInst, type, result, state.glsl_std450, opcode};
|
||||
words.insert(words.end(), args.begin(), args.end());
|
||||
state.builder.AddFunction(words);
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
|
||||
const auto trunc = EmitTruncF32Value(state, src);
|
||||
const auto converted_raw = state.builder.AllocateId();
|
||||
if (signed_value) {
|
||||
const auto converted_i = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertFToS, state.int_type, converted_i, trunc});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, converted_raw, converted_i});
|
||||
} else {
|
||||
state.builder.AddFunction({OpConvertFToU, state.uint_type, converted_raw, trunc});
|
||||
}
|
||||
const auto nan = EmitClassifyF32(state, src).nan;
|
||||
if (signed_value) {
|
||||
const auto below = NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src,
|
||||
ConstantF32(state, 0xcf000000u));
|
||||
const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
|
||||
ConstantF32(state, 0x4f000000u));
|
||||
const auto high = NewSelect(state, state.uint_type, above, ConstantU32(state, 0x7fffffffu),
|
||||
converted_raw);
|
||||
const auto low =
|
||||
NewSelect(state, state.uint_type, below, ConstantU32(state, 0x80000000u), high);
|
||||
return NewSelect(state, state.uint_type, nan, ConstantU32(state, 0), low);
|
||||
}
|
||||
const auto below =
|
||||
NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, ConstantF32(state, 0));
|
||||
const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
|
||||
ConstantF32(state, 0x4f800000u));
|
||||
const auto zero = NewBinary(state, OpLogicalOr, state.bool_type, nan, below);
|
||||
const auto high =
|
||||
NewSelect(state, state.uint_type, above, ConstantU32(state, 0xffffffffu), converted_raw);
|
||||
return NewSelect(state, state.uint_type, zero, ConstantU32(state, 0), high);
|
||||
}
|
||||
|
||||
uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
|
||||
return EmitExt(state, state.uint_type, GlslPackHalf2x16, {src});
|
||||
}
|
||||
|
||||
uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) {
|
||||
const auto pair = EmitExt(state, state.vec2_float_type, GlslUnpackHalf2x16, {bits});
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.float_type, result, pair, 0});
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& state = ctx.state;
|
||||
const auto op = inst.GetOpcode();
|
||||
const auto unary = [&](uint32_t spirv, IR::Type type) {
|
||||
ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0)});
|
||||
return true;
|
||||
};
|
||||
const auto binary = [&](uint32_t spirv, IR::Type type) {
|
||||
ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
|
||||
return true;
|
||||
};
|
||||
const auto ext_unary = [&](uint32_t ext) {
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, ext, {ctx.Arg(inst, 0)}));
|
||||
return true;
|
||||
};
|
||||
switch (op) {
|
||||
case IR::ValueOpcode::BitCastU16F16:
|
||||
case IR::ValueOpcode::BitCastF16U16:
|
||||
case IR::ValueOpcode::ConvertU32U16:
|
||||
case IR::ValueOpcode::ConvertU32U8:
|
||||
case IR::ValueOpcode::PackUint2x32:
|
||||
case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
|
||||
case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
|
||||
case IR::ValueOpcode::ConvertU16U32:
|
||||
ctx.Emit(inst, OpBitwiseAnd, IR::Type::U16,
|
||||
{ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)});
|
||||
return true;
|
||||
case IR::ValueOpcode::ConvertU8U32:
|
||||
ctx.Emit(inst, OpBitwiseAnd, IR::Type::U8,
|
||||
{ctx.Arg(inst, 0), ConstantU32(state, 0xffu)});
|
||||
return true;
|
||||
case IR::ValueOpcode::ConvertF16F32: {
|
||||
const auto pair = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair,
|
||||
ctx.Arg(inst, 0), ConstantF32(state, 0)});
|
||||
ctx.Define(inst, EmitPackHalf(state, pair));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::ConvertF32F16:
|
||||
ctx.Define(inst, EmitUnpackHalf(state, ctx.Arg(inst, 0)));
|
||||
return true;
|
||||
case IR::ValueOpcode::ConvertS32F32:
|
||||
ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::ConvertU32F32:
|
||||
ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::ConvertF32S32: {
|
||||
const auto signed_value = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0));
|
||||
ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value});
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
|
||||
case IR::ValueOpcode::CompositeConstructU32x2:
|
||||
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
|
||||
return true;
|
||||
case IR::ValueOpcode::CompositeConstructF32x2:
|
||||
ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
|
||||
return true;
|
||||
case IR::ValueOpcode::CompositeConstructU32x4:
|
||||
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
|
||||
return true;
|
||||
case IR::ValueOpcode::CompositeExtractU32x2:
|
||||
case IR::ValueOpcode::CompositeExtractU32x4:
|
||||
ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
|
||||
{ctx.Arg(inst, 0), inst.Arg(1).U32()});
|
||||
return true;
|
||||
case IR::ValueOpcode::PackHalf2x16:
|
||||
ctx.Define(inst, EmitExt(state, state.uint_type, GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
|
||||
return true;
|
||||
case IR::ValueOpcode::PackSnorm2x16:
|
||||
ctx.Define(inst,
|
||||
EmitExt(state, state.uint_type, GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
|
||||
return true;
|
||||
case IR::ValueOpcode::PackUnorm2x16:
|
||||
ctx.Define(inst,
|
||||
EmitExt(state, state.uint_type, GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
|
||||
return true;
|
||||
case IR::ValueOpcode::PackFloat2x16Rtz: {
|
||||
const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
|
||||
const auto high =
|
||||
NewBinary(state, OpShiftLeftLogical, state.uint_type,
|
||||
EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
|
||||
ctx.Define(inst, NewBinary(state, OpBitwiseOr, state.uint_type, low, high));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FPAbs32:
|
||||
ctx.Define(inst, EmitFAbsValue(state, ctx.Arg(inst, 0)));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPNeg32:
|
||||
ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPSaturate32:
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, GlslFClamp,
|
||||
{ctx.Arg(inst, 0), ConstantF32(state, 0),
|
||||
ConstantF32(state, 0x3f800000u)}));
|
||||
return true;
|
||||
case IR::ValueOpcode::BitFieldInsert:
|
||||
ctx.Emit(inst, OpBitFieldInsert, IR::Type::U32,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
|
||||
return true;
|
||||
case IR::ValueOpcode::BitFieldUExtract:
|
||||
ctx.Emit(inst, OpBitFieldUExtract, IR::Type::U32,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
|
||||
return true;
|
||||
case IR::ValueOpcode::BitFieldSExtract:
|
||||
ctx.Emit(inst, OpBitFieldSExtract, IR::Type::U32,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
|
||||
return true;
|
||||
case IR::ValueOpcode::SelectU1:
|
||||
ctx.Emit(inst, OpSelect, IR::Type::U1,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
|
||||
return true;
|
||||
case IR::ValueOpcode::SelectU32:
|
||||
ctx.Emit(inst, OpSelect, IR::Type::U32,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
|
||||
return true;
|
||||
case IR::ValueOpcode::SelectF32:
|
||||
ctx.Emit(inst, OpSelect, IR::Type::F32,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
|
||||
return true;
|
||||
case IR::ValueOpcode::IAdd32: return binary(OpIAdd, IR::Type::U32);
|
||||
case IR::ValueOpcode::ISub32: return binary(OpISub, IR::Type::U32);
|
||||
case IR::ValueOpcode::IMul32: return binary(OpIMul, IR::Type::U32);
|
||||
case IR::ValueOpcode::UDiv32: return binary(OpUDiv, IR::Type::U32);
|
||||
case IR::ValueOpcode::IAdd64:
|
||||
ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::ISub64:
|
||||
ctx.Define(inst, EmitSub64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::IMul64:
|
||||
ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::IAddCarry32:
|
||||
ctx.Emit(inst, OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
|
||||
return true;
|
||||
case IR::ValueOpcode::SMulHi:
|
||||
ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::UMulHi:
|
||||
ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::IAbs32: {
|
||||
const auto value = ctx.Arg(inst, 0);
|
||||
const auto neg = NewUnary(state, OpSNegate, state.uint_type, value);
|
||||
const auto negative =
|
||||
NewBinary(state, OpSLessThan, state.bool_type, value, ConstantU32(state, 0));
|
||||
ctx.Define(inst, NewSelect(state, state.uint_type, negative, neg, value));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32);
|
||||
case IR::ValueOpcode::ShiftRightLogical32:
|
||||
return binary(OpShiftRightLogical, IR::Type::U32);
|
||||
case IR::ValueOpcode::ShiftRightArithmetic32:
|
||||
return binary(OpShiftRightArithmetic, IR::Type::U32);
|
||||
case IR::ValueOpcode::ShiftLeftLogical64:
|
||||
ctx.Define(inst,
|
||||
EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::ShiftRightLogical64:
|
||||
ctx.Define(inst,
|
||||
EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::ShiftRightArithmetic64:
|
||||
ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
|
||||
ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitwiseOr32: return binary(OpBitwiseOr, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitwiseAnd64:
|
||||
ctx.Define(inst,
|
||||
EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::BitwiseOr64:
|
||||
ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
|
||||
case IR::ValueOpcode::BitCount64: {
|
||||
const auto pair = ExtractPair(state, ctx.Arg(inst, 0));
|
||||
ctx.Define(inst, NewBinary(state, OpIAdd, state.uint_type,
|
||||
NewUnary(state, OpBitCount, state.uint_type, pair.low),
|
||||
NewUnary(state, OpBitCount, state.uint_type, pair.high)));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FindILsb32: {
|
||||
const auto value = EmitExt(state, state.int_type, GlslFindILsb, {ctx.Arg(inst, 0)});
|
||||
ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FindUMsb32: {
|
||||
const auto value = EmitExt(state, state.int_type, GlslFindUMsb, {ctx.Arg(inst, 0)});
|
||||
ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FindUMsb64:
|
||||
ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0)));
|
||||
return true;
|
||||
case IR::ValueOpcode::FindSMsb32: {
|
||||
const auto value =
|
||||
EmitExt(state, state.int_type, 74u,
|
||||
{NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0))});
|
||||
ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::SMin32:
|
||||
ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::SMax32:
|
||||
ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::UMin32:
|
||||
ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::UMax32:
|
||||
ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::SMinTri32:
|
||||
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
ctx.Arg(inst, 2), true, false));
|
||||
return true;
|
||||
case IR::ValueOpcode::SMaxTri32:
|
||||
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
ctx.Arg(inst, 2), true, true));
|
||||
return true;
|
||||
case IR::ValueOpcode::UMinTri32:
|
||||
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
ctx.Arg(inst, 2), false, false));
|
||||
return true;
|
||||
case IR::ValueOpcode::UMaxTri32:
|
||||
ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
ctx.Arg(inst, 2), false, true));
|
||||
return true;
|
||||
case IR::ValueOpcode::SMedTri32:
|
||||
ctx.Define(inst,
|
||||
EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::UMedTri32:
|
||||
ctx.Define(
|
||||
inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::SLessThan32: return binary(OpSLessThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::ULessThan32: return binary(OpULessThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::IEqual32: return binary(OpIEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::SLessThanEqual32: return binary(OpSLessThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::ULessThanEqual32: return binary(OpULessThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::SGreaterThan32: return binary(OpSGreaterThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::UGreaterThan32: return binary(OpUGreaterThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::INotEqual32: return binary(OpINotEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::IEqual64:
|
||||
ctx.Define(inst,
|
||||
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false));
|
||||
return true;
|
||||
case IR::ValueOpcode::INotEqual64:
|
||||
ctx.Define(inst,
|
||||
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true));
|
||||
return true;
|
||||
case IR::ValueOpcode::ULessThan64:
|
||||
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan,
|
||||
OpULessThan));
|
||||
return true;
|
||||
case IR::ValueOpcode::SLessThan64:
|
||||
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan,
|
||||
OpULessThan));
|
||||
return true;
|
||||
case IR::ValueOpcode::UGreaterThan64:
|
||||
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan,
|
||||
OpUGreaterThan));
|
||||
return true;
|
||||
case IR::ValueOpcode::SGreaterThan64:
|
||||
ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan,
|
||||
OpUGreaterThan));
|
||||
return true;
|
||||
case IR::ValueOpcode::ULessThanEqual64: {
|
||||
const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
OpUGreaterThan, OpUGreaterThan);
|
||||
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::SLessThanEqual64: {
|
||||
const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
OpSGreaterThan, OpUGreaterThan);
|
||||
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::UGreaterThanEqual64: {
|
||||
const auto less =
|
||||
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan);
|
||||
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::SGreaterThanEqual64: {
|
||||
const auto less =
|
||||
Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan);
|
||||
ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
|
||||
case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
|
||||
case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::LogicalNot: return unary(OpLogicalNot, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPOrdEqual32: return binary(OpFOrdEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPUnordEqual32: return binary(OpFUnordEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPOrdNotEqual32: return binary(OpFOrdNotEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPUnordNotEqual32: return binary(OpFUnordNotEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPOrdLessThan32: return binary(OpFOrdLessThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPUnordLessThan32: return binary(OpFUnordLessThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPOrdGreaterThan32: return binary(OpFOrdGreaterThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPUnordGreaterThan32:
|
||||
return binary(OpFUnordGreaterThan, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPOrdLessThanEqual32:
|
||||
return binary(OpFOrdLessThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPUnordLessThanEqual32:
|
||||
return binary(OpFUnordLessThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPOrdGreaterThanEqual32:
|
||||
return binary(OpFOrdGreaterThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPUnordGreaterThanEqual32:
|
||||
return binary(OpFUnordGreaterThanEqual, IR::Type::U1);
|
||||
case IR::ValueOpcode::FPIsNan32:
|
||||
ctx.Emit(inst, OpFUnordNotEqual, IR::Type::U1, {ctx.Arg(inst, 0), ctx.Arg(inst, 0)});
|
||||
return true;
|
||||
case IR::ValueOpcode::FPCmpClass32:
|
||||
ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPAdd32: return binary(OpFAdd, IR::Type::F32);
|
||||
case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32);
|
||||
case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32);
|
||||
case IR::ValueOpcode::FPFma32:
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, GlslFma,
|
||||
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPMin32:
|
||||
ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPMax32:
|
||||
ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPMinTri32:
|
||||
ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
ctx.Arg(inst, 2), false));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPMaxTri32:
|
||||
ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
|
||||
ctx.Arg(inst, 2), true));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPMedTri32:
|
||||
ctx.Define(inst,
|
||||
EmitFMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPRecip32: {
|
||||
const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
|
||||
ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
|
||||
ConstantF32(state, 0x3f800000u), source));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FPRecipIFlag32:
|
||||
// Integer-to-float inputs used by IFLAG cannot be denormal.
|
||||
ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
|
||||
ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPRecipSqrt32:
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, GlslInverseSqrt,
|
||||
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPSqrt:
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, GlslSqrt,
|
||||
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPSin:
|
||||
case IR::ValueOpcode::FPCos: {
|
||||
auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
|
||||
source =
|
||||
NewBinary(state, OpFMul, state.float_type, source, ConstantF32(state, 0x40c90fdbu));
|
||||
ctx.Define(inst, EmitExt(state, state.float_type,
|
||||
op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source}));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FPExp2:
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, GlslExp2,
|
||||
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPLog2:
|
||||
ctx.Define(inst, EmitExt(state, state.float_type, GlslLog2,
|
||||
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
|
||||
return true;
|
||||
case IR::ValueOpcode::FPLdexp: {
|
||||
const auto exponent = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 1));
|
||||
ctx.Define(inst,
|
||||
EmitExt(state, state.float_type, GlslLdexp, {ctx.Arg(inst, 0), exponent}));
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven);
|
||||
case IR::ValueOpcode::FPFloor32: return ext_unary(GlslFloor);
|
||||
case IR::ValueOpcode::FPCeil32: return ext_unary(GlslCeil);
|
||||
case IR::ValueOpcode::FPTrunc32: return ext_unary(GlslTrunc);
|
||||
case IR::ValueOpcode::FPFract32: return ext_unary(GlslFract);
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,557 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value,
|
||||
ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
|
||||
return result;
|
||||
}
|
||||
|
||||
void StoreRegisterMirror(ValueEmitContext& ctx, IR::Register reg, uint32_t value) {
|
||||
if (const auto pointer = PointerForRegister(ctx.state, reg); pointer != 0) {
|
||||
if (IsInactiveWave32ExecHigh(ctx.state, reg)) {
|
||||
value = ConstantU32(ctx.state, 0);
|
||||
}
|
||||
ctx.state.builder.AddFunction({OpStore, pointer, value});
|
||||
}
|
||||
}
|
||||
|
||||
bool IsPrologue(const ValueEmitContext& ctx) {
|
||||
return !ctx.program.blocks.empty() && ctx.current_block == ctx.program.blocks.front();
|
||||
}
|
||||
|
||||
uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
|
||||
auto& state = ctx.state;
|
||||
if (kind == IR::StageInputKind::LocalInvocationIndex) {
|
||||
return EmitLocalInvocationIndex(state);
|
||||
}
|
||||
const auto variable = InputVariableForKind(state, kind);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
if (kind == IR::StageInputKind::FrontFacing) {
|
||||
const auto value = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.bool_type, value, variable});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
return bits;
|
||||
}
|
||||
if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) {
|
||||
const auto value = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.int_type, value, variable});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
|
||||
return bits;
|
||||
}
|
||||
if (kind == IR::StageInputKind::FragCoord) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_input_float, pointer, variable,
|
||||
ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
|
||||
return bits;
|
||||
}
|
||||
return EmitInputComponentU32(state, kind, component);
|
||||
}
|
||||
|
||||
uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
|
||||
auto& state = ctx.state;
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), active});
|
||||
const auto low = state.builder.AllocateId();
|
||||
const auto high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
|
||||
const auto wqm_low = EmitWqmLaneU32(state, low);
|
||||
const auto wqm_high = EmitWqmLaneU32(state, high);
|
||||
const auto lane = EmitSubgroupLocalInvocationId(state);
|
||||
const auto upper = state.builder.AllocateId();
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto bit_lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, mask, upper, wqm_high, wqm_low});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, bit_lane, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_lane});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, result, hit, ConstantU32(state, 0)});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
|
||||
uint32_t exec) {
|
||||
auto& state = ctx.state;
|
||||
const auto lane = EmitSubgroupLocalInvocationId(state);
|
||||
const auto bank_shift = state.builder.AllocateId();
|
||||
const auto row_shift = state.builder.AllocateId();
|
||||
const auto bank = state.builder.AllocateId();
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto bank_bit = state.builder.AllocateId();
|
||||
const auto row_bit = state.builder.AllocateId();
|
||||
const auto bank_hit = state.builder.AllocateId();
|
||||
const auto row_hit = state.builder.AllocateId();
|
||||
const auto bank_ok = state.builder.AllocateId();
|
||||
const auto row_ok = state.builder.AllocateId();
|
||||
const auto masks_ok = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, bank_shift, lane, ConstantU32(state, 2)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, row_shift, lane, ConstantU32(state, 4)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, flags.row_mask), row_bit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, bank_ok, bank_hit, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, row_ok, row_hit, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, masks_ok, bank_ok, row_ok});
|
||||
uint32_t writable = masks_ok;
|
||||
if (!flags.bound_control) {
|
||||
const auto target = EmitDppTargetLane(state, flags.control);
|
||||
const auto bounded = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, bounded, writable, target.valid});
|
||||
writable = bounded;
|
||||
}
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, result, exec, writable});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
|
||||
auto& state = ctx.state;
|
||||
const auto* input = InputBindingForParameter(state, attr);
|
||||
if (input == nullptr || input->variable_id == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
if (state.stage == ShaderType::Vertex) {
|
||||
return EmitVertexParameterComponentU32(state, *input, chan & 3u);
|
||||
}
|
||||
const auto vector = state.builder.AllocateId();
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.vec4_float_type, vector, input->variable_id});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.float_type, component, vector, chan & 3u});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
return bits;
|
||||
}
|
||||
|
||||
bool MrtUsesUint(const EmitterState& state, const IR::ExportInfo& exp) {
|
||||
return exp.kind == IR::ExportTargetKind::Mrt && state.pixel_input_info != nullptr &&
|
||||
exp.index < std::size(state.pixel_input_info->target_output_mode) &&
|
||||
state.pixel_input_info->target_output_mode[exp.index] == 7u;
|
||||
}
|
||||
|
||||
uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
|
||||
const auto value = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeExtract, ctx.state.uint_type, value, vector, component});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp,
|
||||
bool uint_output) {
|
||||
auto& state = ctx.state;
|
||||
if (exp.compr && !uint_output) {
|
||||
uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
|
||||
ConstantF32(state, 0x3f800000u)};
|
||||
for (uint32_t pair = 0; pair < 2u; pair++) {
|
||||
if ((exp.en & (3u << (pair * 2u))) == 0u) {
|
||||
continue;
|
||||
}
|
||||
const auto packed = state.builder.AllocateId();
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, packed, data, pair});
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
|
||||
state.glsl_std450, GlslUnpackHalf2x16, packed});
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair * 2u + lane;
|
||||
if (((exp.en >> component) & 1u) != 0u) {
|
||||
f32[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, f32[component], unpacked, lane});
|
||||
}
|
||||
}
|
||||
}
|
||||
const auto vector = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
|
||||
return vector;
|
||||
}
|
||||
uint32_t raw[4] = {
|
||||
ConstantU32(state, 0),
|
||||
ConstantU32(state, 0),
|
||||
ConstantU32(state, 0),
|
||||
ConstantU32(state, uint_output ? 1u : 0x3f800000u),
|
||||
};
|
||||
if (exp.compr) {
|
||||
for (uint32_t pair = 0; pair < 2u; pair++) {
|
||||
if ((exp.en & (3u << (pair * 2u))) == 0u) {
|
||||
continue;
|
||||
}
|
||||
const auto packed = ExportRawComponent(ctx, data, pair);
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair * 2u + lane;
|
||||
if (((exp.en >> component) & 1u) == 0u) {
|
||||
continue;
|
||||
}
|
||||
raw[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, raw[component],
|
||||
packed, ConstantU32(state, lane * 16u),
|
||||
ConstantU32(state, 16)});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (uint32_t component = 0; component < 4u; component++) {
|
||||
if (((exp.en >> component) & 1u) != 0u) {
|
||||
raw[component] = ExportRawComponent(ctx, data, component);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (uint_output) {
|
||||
const auto vector = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec4_uint_type, vector, raw[0], raw[1], raw[2], raw[3]});
|
||||
return vector;
|
||||
}
|
||||
uint32_t f32[4] {};
|
||||
for (uint32_t component = 0; component < 4u; component++) {
|
||||
f32[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32[component], raw[component]});
|
||||
}
|
||||
const auto vector = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
|
||||
return vector;
|
||||
}
|
||||
|
||||
void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& state = ctx.state;
|
||||
const auto& exp = ctx.Export(inst);
|
||||
const auto exec = ctx.Arg(inst, 1);
|
||||
if (state.stage == ShaderType::Pixel && exp.vm && state.needs_pixel_valid_mask &&
|
||||
state.pixel_valid_mask_variable != 0) {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
|
||||
}
|
||||
if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
|
||||
exp.en == 0u) {
|
||||
return;
|
||||
}
|
||||
EmitIfCondition(state, exec, [&]() {
|
||||
const auto data = ctx.Arg(inst, 0);
|
||||
if (exp.kind == IR::ExportTargetKind::MrtZ) {
|
||||
if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
|
||||
const auto raw = ExportRawComponent(ctx, data, 0);
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, raw});
|
||||
state.builder.AddFunction({OpStore, state.depth_variable, f32});
|
||||
}
|
||||
if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
|
||||
const auto raw = ExportRawComponent(ctx, data, 2);
|
||||
const auto value = state.builder.AllocateId();
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, value, raw});
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, pointer,
|
||||
state.sample_mask_variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, pointer, value});
|
||||
}
|
||||
return;
|
||||
}
|
||||
const auto variable = OutputVariableForExport(state, exp);
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
const bool uint_output = MrtUsesUint(state, exp);
|
||||
const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
|
||||
auto value = ExportVector(ctx, data, exp, uint_output);
|
||||
if (exp.kind == IR::ExportTargetKind::Mrt && state.pixel_input_info != nullptr &&
|
||||
exp.index < state.pixel_input_info->target_export_mapping.size()) {
|
||||
const auto mapping = state.pixel_input_info->target_export_mapping[exp.index];
|
||||
if (!mapping.IsIdentity()) {
|
||||
const auto mapped = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
|
||||
mapping.Map(0), mapping.Map(1), mapping.Map(2),
|
||||
mapping.Map(3)});
|
||||
value = mapped;
|
||||
}
|
||||
}
|
||||
if (exp.kind == IR::ExportTargetKind::Position) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer,
|
||||
variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, pointer, value});
|
||||
} else {
|
||||
state.builder.AddFunction({OpStore, variable, value});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& state = ctx.state;
|
||||
switch (inst.GetOpcode()) {
|
||||
case IR::ValueOpcode::Identity:
|
||||
case IR::ValueOpcode::ConditionRef: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
|
||||
case IR::ValueOpcode::SetScalarRegister:
|
||||
StoreRegisterMirror(
|
||||
ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
|
||||
ctx.Arg(inst, 1));
|
||||
return true;
|
||||
case IR::ValueOpcode::SetThreadBitScalarRegister:
|
||||
StoreRegisterMirror(
|
||||
ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
|
||||
BoolAsU32(ctx, ctx.Arg(inst, 1)));
|
||||
return true;
|
||||
case IR::ValueOpcode::SetVectorRegister:
|
||||
StoreRegisterMirror(
|
||||
ctx, {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())},
|
||||
ctx.Arg(inst, 1));
|
||||
return true;
|
||||
case IR::ValueOpcode::SetScc: ctx.Arg(inst, 0); return true;
|
||||
case IR::ValueOpcode::SetExec: ctx.Arg(inst, 0); return true;
|
||||
case IR::ValueOpcode::SetExecLo:
|
||||
case IR::ValueOpcode::SetExecHi:
|
||||
if (!IsPrologue(ctx)) {
|
||||
StoreRegisterMirror(ctx,
|
||||
{IR::RegisterFile::Exec,
|
||||
inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u},
|
||||
ctx.Arg(inst, 0));
|
||||
}
|
||||
return true;
|
||||
case IR::ValueOpcode::SetVcc: ctx.Arg(inst, 0); return true;
|
||||
case IR::ValueOpcode::SetVccLo:
|
||||
case IR::ValueOpcode::SetVccHi:
|
||||
if (!IsPrologue(ctx)) {
|
||||
StoreRegisterMirror(ctx,
|
||||
{IR::RegisterFile::Vcc,
|
||||
inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u},
|
||||
ctx.Arg(inst, 0));
|
||||
}
|
||||
return true;
|
||||
case IR::ValueOpcode::SetM0:
|
||||
StoreRegisterMirror(ctx, {IR::RegisterFile::M0, 0}, ctx.Arg(inst, 0));
|
||||
return true;
|
||||
case IR::ValueOpcode::GetScalarRegister:
|
||||
case IR::ValueOpcode::GetThreadBitScalarRegister:
|
||||
case IR::ValueOpcode::GetVectorRegister:
|
||||
case IR::ValueOpcode::GetScc:
|
||||
case IR::ValueOpcode::GetExec:
|
||||
case IR::ValueOpcode::GetExecLo:
|
||||
case IR::ValueOpcode::GetExecHi:
|
||||
case IR::ValueOpcode::GetVcc:
|
||||
case IR::ValueOpcode::GetVccLo:
|
||||
case IR::ValueOpcode::GetVccHi:
|
||||
case IR::ValueOpcode::GetM0: return true;
|
||||
case IR::ValueOpcode::Void:
|
||||
case IR::ValueOpcode::Reference:
|
||||
case IR::ValueOpcode::ReferenceU32:
|
||||
case IR::ValueOpcode::Prologue:
|
||||
case IR::ValueOpcode::Epilogue:
|
||||
case IR::ValueOpcode::ControlNop:
|
||||
case IR::ValueOpcode::Waitcnt:
|
||||
case IR::ValueOpcode::Sendmsg:
|
||||
case IR::ValueOpcode::TtraceData:
|
||||
case IR::ValueOpcode::InstPrefetch: return true;
|
||||
case IR::ValueOpcode::Barrier: {
|
||||
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
|
||||
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
|
||||
ConstantU32(state, ScopeWorkgroup),
|
||||
ConstantU32(state, semantics)});
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::GetUserData: {
|
||||
const auto reg = inst.Arg(0).ScalarRegister();
|
||||
uint32_t dword = 0;
|
||||
if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) {
|
||||
ctx.Define(inst, ConstantU32(state, 0));
|
||||
} else {
|
||||
ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::GetBuiltin:
|
||||
ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
|
||||
inst.Arg(1).U32()));
|
||||
return true;
|
||||
case IR::ValueOpcode::UndefU1:
|
||||
case IR::ValueOpcode::UndefU8:
|
||||
case IR::ValueOpcode::UndefU16:
|
||||
case IR::ValueOpcode::UndefU32:
|
||||
case IR::ValueOpcode::UndefU64:
|
||||
state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
|
||||
return true;
|
||||
case IR::ValueOpcode::DppMoveU32: {
|
||||
const auto flags = inst.Flags<IR::DppMoveFlags>();
|
||||
IR::Operand operand;
|
||||
operand.dpp = true;
|
||||
operand.dpp_ctrl = flags.control;
|
||||
operand.dpp_row_mask = flags.row_mask;
|
||||
operand.dpp_bank_mask = flags.bank_mask;
|
||||
operand.dpp_fetch_inactive = flags.fetch_inactive;
|
||||
operand.dpp_bound_ctrl = flags.bound_control;
|
||||
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
|
||||
target.lane});
|
||||
if (operand.dpp_fetch_inactive) {
|
||||
ctx.Define(inst, shuffled);
|
||||
return true;
|
||||
}
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
|
||||
const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
|
||||
const auto can_fetch = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
|
||||
ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::DppUpdateU32: {
|
||||
const auto flags = inst.Flags<IR::DppMoveFlags>();
|
||||
const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
|
||||
ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::WqmMask:
|
||||
ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
|
||||
return true;
|
||||
case IR::ValueOpcode::LaneId:
|
||||
ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
|
||||
return true;
|
||||
case IR::ValueOpcode::Ballot:
|
||||
if (state.per_invocation_masks) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result,
|
||||
BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
|
||||
ConstantU32(state, 0), ConstantU32(state, 0)});
|
||||
ctx.Define(inst, result);
|
||||
} else {
|
||||
ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
|
||||
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
|
||||
}
|
||||
return true;
|
||||
case IR::ValueOpcode::ReadFirstLane: {
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
|
||||
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, lane,
|
||||
ConstantU32(state, ScopeSubgroup), ballot});
|
||||
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
|
||||
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::ReadLane:
|
||||
ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
|
||||
{ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
|
||||
return true;
|
||||
case IR::ValueOpcode::WriteLane: {
|
||||
const auto hit = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpIEqual, state.bool_type, hit,
|
||||
EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
|
||||
ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::Permlane16U32: {
|
||||
const auto flags = inst.Flags<IR::PermlaneFlags>();
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto row_value = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane8 = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto upper = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto shifted = state.builder.AllocateId();
|
||||
const auto index = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
if (flags.x16) {
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCopyObject, state.uint_type, row_value, row});
|
||||
}
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, shifted, selected, shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, index, shifted, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index});
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
|
||||
target});
|
||||
uint32_t result = shuffled;
|
||||
if (!flags.fetch_inactive) {
|
||||
const auto source_exec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
|
||||
target});
|
||||
result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, result, source_exec, shuffled,
|
||||
ConstantU32(state, 0)});
|
||||
}
|
||||
ctx.Define(inst, result);
|
||||
return true;
|
||||
}
|
||||
case IR::ValueOpcode::GetAttribute:
|
||||
ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
|
||||
return true;
|
||||
case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
|
||||
case IR::ValueOpcode::GetShaderBase:
|
||||
// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
|
||||
// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
|
||||
ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
|
||||
return true;
|
||||
case IR::ValueOpcode::GetSrtResource:
|
||||
case IR::ValueOpcode::GetBufferResource:
|
||||
case IR::ValueOpcode::GetAddressResource:
|
||||
case IR::ValueOpcode::GetImageResource:
|
||||
case IR::ValueOpcode::GetSamplerResource:
|
||||
case IR::ValueOpcode::GetLdsResource:
|
||||
case IR::ValueOpcode::GetGdsResource:
|
||||
case IR::ValueOpcode::MakeImageAddress: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,516 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, value});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, lhs, rhs});
|
||||
return result;
|
||||
}
|
||||
|
||||
bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {
|
||||
return (mem.image_sample_flags & flag) != 0u;
|
||||
}
|
||||
|
||||
bool UsesA16(const IR::MemoryInfo& mem) {
|
||||
return HasFlag(mem, Decoder::ImageSampleFlagA16);
|
||||
}
|
||||
|
||||
bool ComponentUsesA16(const IR::MemoryInfo& mem, uint32_t component) {
|
||||
if (!UsesA16(mem)) return false;
|
||||
uint32_t cursor = 0;
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) {
|
||||
if (component == cursor) return false;
|
||||
cursor++;
|
||||
}
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagBias)) {
|
||||
if (component == cursor) return true;
|
||||
cursor++;
|
||||
}
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagCompare) && component == cursor) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t HalfComponent(const IR::MemoryInfo& mem, uint32_t component) {
|
||||
if (!UsesA16(mem)) return component * 2u;
|
||||
uint32_t half = 0;
|
||||
for (uint32_t index = 0; index < component; index++) {
|
||||
const auto width = ComponentUsesA16(mem, index) ? 1u : 2u;
|
||||
if (width == 2u && (half & 1u) != 0u) half++;
|
||||
half += width;
|
||||
}
|
||||
if (!ComponentUsesA16(mem, component) && (half & 1u) != 0u) half++;
|
||||
return half;
|
||||
}
|
||||
|
||||
uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
|
||||
uint32_t component) {
|
||||
const auto half = HalfComponent(mem, component);
|
||||
const auto packed = half / 2u;
|
||||
if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0);
|
||||
auto value = ctx.Def(address.Arg(packed));
|
||||
if (ComponentUsesA16(mem, component)) {
|
||||
if ((half & 1u) != 0u) {
|
||||
value = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, value,
|
||||
ConstantU32(ctx.state, 16));
|
||||
}
|
||||
value = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, value,
|
||||
ConstantU32(ctx.state, 0xffffu));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
|
||||
uint32_t component) {
|
||||
const auto value = AddressU32(ctx, mem, address, component);
|
||||
return ComponentUsesA16(mem, component)
|
||||
? EmitF16BitsToF32(ctx.state, value)
|
||||
: Unary(ctx.state, OpBitcast, ctx.state.float_type, value);
|
||||
}
|
||||
|
||||
ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) {
|
||||
ImageSampleLayout layout;
|
||||
uint32_t cursor = 0;
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++;
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++;
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++;
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
|
||||
layout.grad_x = cursor;
|
||||
cursor += ImageViewSpatialComponents(view);
|
||||
layout.grad_y = cursor;
|
||||
cursor += ImageViewSpatialComponents(view);
|
||||
}
|
||||
layout.coord = cursor;
|
||||
cursor += ImageViewCoordinateComponents(view);
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagLod)) layout.lod = cursor++;
|
||||
return layout;
|
||||
}
|
||||
|
||||
uint32_t ZeroF32(EmitterState& state) {
|
||||
return ConstantF32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t ZeroU32x4(EmitterState& state) {
|
||||
const auto zero = ConstantU32(state, 0);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType(
|
||||
{OpConstantComposite, state.vec4_uint_type, value, zero, zero, zero, zero});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t CubeAxis(EmitterState& state, uint32_t value) {
|
||||
return Binary(state, OpFSub, state.float_type, value, ConstantF32(state, 0x3f800000u));
|
||||
}
|
||||
|
||||
uint32_t CubeLayer(EmitterState& state, uint32_t value) {
|
||||
const auto guest = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertFToU, state.uint_type, guest, value});
|
||||
const auto padding =
|
||||
Binary(state, OpShiftLeftLogical, state.uint_type,
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, guest, ConstantU32(state, 3)),
|
||||
ConstantU32(state, 1));
|
||||
const auto host = Binary(state, OpISub, state.uint_type, guest, padding);
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpConvertUToF, state.float_type, result, host});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
|
||||
uint32_t first, uint32_t components) {
|
||||
auto x = AddressF32(ctx, mem, address, first);
|
||||
if (components == 1u) return x;
|
||||
auto y = mem.image_address_components > first + 1u ? AddressF32(ctx, mem, address, first + 1u)
|
||||
: ZeroF32(ctx.state);
|
||||
if (mem.image_cube) {
|
||||
x = CubeAxis(ctx.state, x);
|
||||
y = CubeAxis(ctx.state, y);
|
||||
}
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
auto z = mem.image_address_components > first + 2u
|
||||
? AddressF32(ctx, mem, address, first + 2u)
|
||||
: ZeroF32(ctx.state);
|
||||
if (mem.image_cube) z = CubeLayer(ctx.state, z);
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec3_float_type, result, x, y, z});
|
||||
} else {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec2_float_type, result, x, y});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
|
||||
ImageViewKind view) {
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
const auto x = AddressU32(ctx, mem, address, 0);
|
||||
if (components == 1u) return x;
|
||||
const auto y = mem.image_address_components > 1u ? AddressU32(ctx, mem, address, 1)
|
||||
: ConstantU32(ctx.state, 0);
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
|
||||
: ConstantU32(ctx.state, 0);
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec3_uint_type, result, x, y, z});
|
||||
} else {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec2_uint_type, result, x, y});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
|
||||
ImageViewKind view) {
|
||||
const auto component = ImageViewCoordinateComponents(view);
|
||||
return mem.image_has_mip && mem.image_address_components > component
|
||||
? AddressU32(ctx, mem, address, component)
|
||||
: ConstantU32(ctx.state, 0);
|
||||
}
|
||||
|
||||
uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
|
||||
return Unary(ctx.state, OpBitcast, ctx.state.uint_type, value);
|
||||
}
|
||||
|
||||
uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) {
|
||||
uint32_t component[4] {};
|
||||
for (uint32_t index = 0; index < 4u; index++) {
|
||||
if (dref) {
|
||||
component[index] = index == 0u ? FloatBits(ctx, value) : ConstantU32(ctx.state, 0);
|
||||
continue;
|
||||
}
|
||||
const auto scalar = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpCompositeExtract,
|
||||
integer ? ctx.state.uint_type : ctx.state.float_type, scalar,
|
||||
value, index});
|
||||
component[index] = integer ? scalar : FloatBits(ctx, scalar);
|
||||
}
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, result,
|
||||
component[0], component[1], component[2], component[3]});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const IR::Inst& address) {
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
const auto view = SampledImageViewKind(ctx.state, mem, pc);
|
||||
const auto image = LoadSampledImageDescriptor(ctx.state, mem, pc, view);
|
||||
const auto size = ctx.state.builder.AllocateId();
|
||||
if (ImageSpirvMultisampled(view) != 0u) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpImageQuerySize, ImageViewSizeType(ctx.state, view), size, image});
|
||||
} else {
|
||||
ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, view),
|
||||
size, image, AddressU32(ctx, mem, address, 0)});
|
||||
}
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
uint32_t result[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
|
||||
ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
|
||||
if (components == 1u) {
|
||||
result[0] = size;
|
||||
} else {
|
||||
for (uint32_t index = 0; index < components; index++) {
|
||||
result[index] = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeExtract, ctx.state.uint_type, result[index], size, index});
|
||||
}
|
||||
}
|
||||
if (ImageSpirvMultisampled(view) == 0u) {
|
||||
result[3] = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpImageQueryLevels, ctx.state.uint_type, result[3], image});
|
||||
}
|
||||
const auto vector = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, vector,
|
||||
result[0], result[1], result[2], result[3]});
|
||||
return vector;
|
||||
}
|
||||
|
||||
uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
const auto zero = ConstantI32(ctx.state, 0);
|
||||
if (layout.offset == NoImageComponent || mem.image_address_components <= layout.offset) {
|
||||
if (components == 1u) return zero;
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec3_int_type, result, zero, zero, zero});
|
||||
} else {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec2_int_type, result, zero, zero});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
const auto packed = Unary(ctx.state, OpBitcast, ctx.state.int_type,
|
||||
AddressU32(ctx, mem, address, layout.offset));
|
||||
uint32_t values[3] = {zero, zero, zero};
|
||||
for (uint32_t index = 0; index < components; index++) {
|
||||
values[index] = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpBitFieldSExtract, ctx.state.int_type, values[index],
|
||||
packed, ConstantU32(ctx.state, index * 8u),
|
||||
ConstantU32(ctx.state, 6)});
|
||||
}
|
||||
if (components == 1u) return values[0];
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec3_int_type, result,
|
||||
values[0], values[1], values[2]});
|
||||
} else {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeConstruct, ctx.state.vec2_int_type, result, values[0], values[1]});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
const auto array_type = state.builder.AllocateId();
|
||||
const auto count = ConstantU32(state, 4);
|
||||
const auto element_type = components == 1u ? state.int_type : state.vec2_int_type;
|
||||
state.builder.AddType({OpTypeArray, array_type, element_type, count});
|
||||
uint32_t offsets[4] {};
|
||||
for (uint32_t index = 0; index < 4u; index++) {
|
||||
const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
|
||||
if (components == 1u) {
|
||||
offsets[index] = x;
|
||||
} else {
|
||||
offsets[index] = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, state.vec2_int_type, offsets[index], x,
|
||||
ConstantI32(state, 0)});
|
||||
}
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType(
|
||||
{OpConstantComposite, array_type, value, offsets[0], offsets[1], offsets[2], offsets[3]});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
|
||||
for (uint32_t source = 0; source < 4u; source++) {
|
||||
if (((swizzle >> (source * 3u)) & 7u) == 4u + component) return source;
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) {
|
||||
const auto swizzle = ctx.state.program.info.images[mem.resource].storage_swizzle;
|
||||
uint32_t values[4] {};
|
||||
const auto dmask = mem.dmask != 0u ? mem.dmask : 1u;
|
||||
for (uint32_t component = 0; component < 4u; component++) {
|
||||
const auto source = InverseSwizzle(swizzle, component);
|
||||
uint32_t raw = ConstantU32(ctx.state, 0);
|
||||
if (source < 4u && ((dmask >> source) & 1u) != 0u) {
|
||||
const auto packed_index = DmaskComponentIndex(dmask, source);
|
||||
raw = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpCompositeExtract, ctx.state.uint_type, raw, data, packed_index});
|
||||
}
|
||||
values[component] = integer ? raw : Unary(ctx.state, OpBitcast, ctx.state.float_type, raw);
|
||||
}
|
||||
const auto texel = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpCompositeConstruct,
|
||||
integer ? ctx.state.vec4_uint_type : ctx.state.vec4_float_type,
|
||||
texel, values[0], values[1], values[2], values[3]});
|
||||
return texel;
|
||||
}
|
||||
|
||||
uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
|
||||
switch (opcode) {
|
||||
case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
|
||||
case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
|
||||
case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
|
||||
case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
|
||||
case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
|
||||
case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
const auto op = inst.GetOpcode();
|
||||
if (op < IR::ValueOpcode::ImageQueryDimensions || op > IR::ValueOpcode::ImageAtomicXor32) {
|
||||
return false;
|
||||
}
|
||||
auto& state = ctx.state;
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
const auto image_arg = inst.Arg(0);
|
||||
ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
|
||||
const bool sampled = op == IR::ValueOpcode::ImageQueryLod ||
|
||||
op == IR::ValueOpcode::ImageSampleRaw ||
|
||||
op == IR::ValueOpcode::ImageGatherRaw;
|
||||
const auto* address = ctx.ImageAddress(inst.Arg(sampled ? 2 : 1));
|
||||
if (address == nullptr) return true;
|
||||
if (op == IR::ValueOpcode::ImageQueryDimensions) {
|
||||
ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address));
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::ImageQueryLod) {
|
||||
const auto view = SampledImageViewKind(state, mem, pc);
|
||||
const auto sampled = MakeSampledImage(state, mem, pc, view);
|
||||
const auto lod = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpImageQueryLod, state.vec2_float_type, lod, sampled,
|
||||
CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))});
|
||||
uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
|
||||
ConstantU32(state, 0)};
|
||||
for (uint32_t index = 0; index < 2u; index++) {
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, lod, index});
|
||||
values[index] = FloatBits(ctx, component);
|
||||
}
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, values[0],
|
||||
values[1], values[2], values[3]});
|
||||
ctx.Define(inst, result);
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::ImageRead) {
|
||||
const auto view = SampledImageViewKind(state, mem, pc);
|
||||
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
|
||||
const auto condition = ctx.Arg(inst, 2);
|
||||
ctx.Define(
|
||||
inst,
|
||||
EmitValueOrDefaultIfCondition(
|
||||
state, condition, state.vec4_uint_type, ZeroU32x4(state), [&]() {
|
||||
const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
|
||||
const auto color = state.builder.AllocateId();
|
||||
const auto coord = CoordU32(ctx, mem, *address, view);
|
||||
if (ImageSpirvMultisampled(view) != 0u) {
|
||||
state.builder.AddFunction(
|
||||
{OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
|
||||
color, image, coord, ImageOperandsSampleMask,
|
||||
AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
|
||||
color, image, coord, ImageOperandsLodMask,
|
||||
LodU32(ctx, mem, *address, view)});
|
||||
}
|
||||
return ResultVector(ctx, color, integer, false);
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::ImageWrite) {
|
||||
const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint;
|
||||
const auto view = StorageImageViewKind(state, mem, uint_image, pc);
|
||||
EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
|
||||
state.builder.AddFunction(
|
||||
{OpImageWrite,
|
||||
LoadStorageImageDescriptor(state, mem.resource, uint_image, pc, view),
|
||||
CoordU32(ctx, mem, *address, view),
|
||||
StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image)});
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
|
||||
const auto view = SampledImageViewKind(state, mem, pc);
|
||||
const auto layout = Layout(mem, view);
|
||||
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
|
||||
const bool dref = HasFlag(mem, Decoder::ImageSampleFlagCompare);
|
||||
const auto sampled = MakeSampledImage(state, mem, pc, view);
|
||||
const auto coord =
|
||||
CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view));
|
||||
const auto sample = state.builder.AllocateId();
|
||||
if (op == IR::ValueOpcode::ImageGatherRaw) {
|
||||
std::vector<uint32_t> words =
|
||||
dref ? std::vector<uint32_t> {OpImageDrefGather,
|
||||
state.vec4_float_type,
|
||||
sample,
|
||||
sampled,
|
||||
coord,
|
||||
layout.dref != NoImageComponent
|
||||
? AddressF32(ctx, mem, *address, layout.dref)
|
||||
: ZeroF32(state)}
|
||||
: std::vector<uint32_t> {
|
||||
OpImageGather, integer ? state.vec4_uint_type : state.vec4_float_type,
|
||||
sample, sampled,
|
||||
coord, ConstantU32(state, ImageGatherComponent(mem.dmask))};
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
|
||||
words.push_back(ImageOperandsConstOffsetsMask);
|
||||
words.push_back(HorizontalOffsets(state, view));
|
||||
} else if (layout.offset != NoImageComponent) {
|
||||
words.push_back(ImageOperandsOffsetMask);
|
||||
words.push_back(PackedOffset(ctx, mem, *address, layout, view));
|
||||
}
|
||||
state.builder.AddFunction(words);
|
||||
ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false));
|
||||
return true;
|
||||
}
|
||||
const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
|
||||
HasFlag(mem, Decoder::ImageSampleFlagLod) ||
|
||||
HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
|
||||
state.stage != ShaderType::Pixel;
|
||||
const auto opcode = explicit_lod
|
||||
? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod)
|
||||
: (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod);
|
||||
const auto result_type =
|
||||
dref ? state.float_type : (integer ? state.vec4_uint_type : state.vec4_float_type);
|
||||
std::vector<uint32_t> words {opcode, result_type, sample, sampled, coord};
|
||||
if (dref) {
|
||||
words.push_back(layout.dref != NoImageComponent
|
||||
? AddressF32(ctx, mem, *address, layout.dref)
|
||||
: ZeroF32(state));
|
||||
}
|
||||
uint32_t operand_mask = 0;
|
||||
std::vector<uint32_t> operands;
|
||||
if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
|
||||
operand_mask |= ImageOperandsGradMask;
|
||||
operands.push_back(
|
||||
CoordF32(ctx, mem, *address, layout.grad_x, ImageViewSpatialComponents(view)));
|
||||
operands.push_back(
|
||||
CoordF32(ctx, mem, *address, layout.grad_y, ImageViewSpatialComponents(view)));
|
||||
} else if (explicit_lod) {
|
||||
operand_mask |= ImageOperandsLodMask;
|
||||
operands.push_back(HasFlag(mem, Decoder::ImageSampleFlagLod) &&
|
||||
layout.lod != NoImageComponent
|
||||
? AddressF32(ctx, mem, *address, layout.lod)
|
||||
: ZeroF32(state));
|
||||
} else if (layout.bias != NoImageComponent) {
|
||||
operand_mask |= ImageOperandsBiasMask;
|
||||
operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
|
||||
}
|
||||
if (operand_mask != 0u) {
|
||||
words.push_back(operand_mask);
|
||||
words.insert(words.end(), operands.begin(), operands.end());
|
||||
}
|
||||
state.builder.AddFunction(words);
|
||||
ctx.Define(inst, ResultVector(ctx, sample, integer, dref));
|
||||
return true;
|
||||
}
|
||||
const auto atomic_opcode = ImageAtomicOpcode(op);
|
||||
if (atomic_opcode != 0u) {
|
||||
const auto view = StorageImageViewKind(state, mem, true, pc);
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpImageTexelPointer, state.ptr_image_uint, pointer,
|
||||
StorageImageDescriptorPointer(state, mem.resource, true, pc, view),
|
||||
CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)});
|
||||
const auto old = state.builder.AllocateId();
|
||||
state.builder.AddFunction({atomic_opcode, state.uint_type, old, pointer,
|
||||
ConstantU32(state, ScopeDevice),
|
||||
ConstantU32(state, MemorySemanticsNone),
|
||||
ctx.Arg(inst, 2)});
|
||||
EmitDeviceAtomicMemoryBarrier(state);
|
||||
return old;
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,713 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
namespace {
|
||||
|
||||
uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, value});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, type, result, lhs, rhs});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
|
||||
uint32_t false_value) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
|
||||
if (lhs == 0) return rhs;
|
||||
if (rhs == 0) return lhs;
|
||||
return Binary(state, OpLogicalAnd, state.bool_type, lhs, rhs);
|
||||
}
|
||||
|
||||
uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
uint32_t index, uint32_t offset, uint32_t soffset) {
|
||||
auto& state = ctx.state;
|
||||
const auto packed =
|
||||
ConstantU32(state, StorageBufferPackedStride(state, mem, inst.Flags<IR::MemoryFlags>().pc));
|
||||
if (mem.formatted && !mem.typed && inst.GetOpcode() == IR::ValueOpcode::StoreBufferU32) {
|
||||
const auto add_tid_bit = Binary(
|
||||
state, OpBitwiseAnd, state.uint_type,
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 20)),
|
||||
ConstantU32(state, 1));
|
||||
const auto add_tid =
|
||||
Binary(state, OpINotEqual, state.bool_type, add_tid_bit, ConstantU32(state, 0));
|
||||
const auto lane = Binary(state, OpBitwiseAnd, state.uint_type,
|
||||
EmitLocalInvocationIndex(state), ConstantU32(state, 63));
|
||||
index = Select(state, state.uint_type, add_tid,
|
||||
Binary(state, OpIAdd, state.uint_type, index, lane), index);
|
||||
}
|
||||
const auto stride =
|
||||
Binary(state, OpBitwiseAnd, state.uint_type, packed, ConstantU32(state, 0x3fffu));
|
||||
const auto swizzle =
|
||||
Binary(state, OpBitwiseAnd, state.uint_type,
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 14)),
|
||||
ConstantU32(state, 1));
|
||||
const auto use_swizzle =
|
||||
Binary(state, OpLogicalAnd, state.bool_type,
|
||||
Binary(state, OpINotEqual, state.bool_type, stride, ConstantU32(state, 0)),
|
||||
Binary(state, OpINotEqual, state.bool_type, swizzle, ConstantU32(state, 0)));
|
||||
const auto stride_enum =
|
||||
Binary(state, OpBitwiseAnd, state.uint_type,
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 16)),
|
||||
ConstantU32(state, 3));
|
||||
const auto index_shift =
|
||||
Binary(state, OpIAdd, state.uint_type, stride_enum, ConstantU32(state, 3));
|
||||
const auto index_stride =
|
||||
Binary(state, OpShiftLeftLogical, state.uint_type, ConstantU32(state, 8), stride_enum);
|
||||
const auto index_mask =
|
||||
Binary(state, OpISub, state.uint_type, index_stride, ConstantU32(state, 1));
|
||||
offset = Binary(state, OpIAdd, state.uint_type, offset, ConstantU32(state, mem.offset));
|
||||
const auto linear = Binary(state, OpIAdd, state.uint_type,
|
||||
Binary(state, OpIMul, state.uint_type, index, stride), offset);
|
||||
const auto index_msb = Binary(state, OpShiftRightLogical, state.uint_type, index, index_shift);
|
||||
const auto index_lsb = Binary(state, OpBitwiseAnd, state.uint_type, index, index_mask);
|
||||
const auto offset_msb =
|
||||
Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, ~3u));
|
||||
const auto offset_lsb =
|
||||
Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, 3));
|
||||
const auto msb =
|
||||
Binary(state, OpIMul, state.uint_type,
|
||||
Binary(state, OpIAdd, state.uint_type,
|
||||
Binary(state, OpIMul, state.uint_type, index_msb, stride), offset_msb),
|
||||
index_stride);
|
||||
const auto lsb =
|
||||
Binary(state, OpIAdd, state.uint_type,
|
||||
Binary(state, OpShiftLeftLogical, state.uint_type, index_lsb, ConstantU32(state, 2)),
|
||||
offset_lsb);
|
||||
const auto address = Select(state, state.uint_type, use_swizzle,
|
||||
Binary(state, OpIAdd, state.uint_type, msb, lsb), linear);
|
||||
return Binary(state, OpIAdd, state.uint_type, address, soffset);
|
||||
}
|
||||
|
||||
uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low,
|
||||
uint32_t add_high, uint32_t& out_high) {
|
||||
const auto result = Binary(state, OpIAdd, state.uint_type, low, add_low);
|
||||
const auto carry = Binary(state, OpULessThan, state.bool_type, result, low);
|
||||
out_high = Binary(
|
||||
state, OpIAdd, state.uint_type, Binary(state, OpIAdd, state.uint_type, high, add_high),
|
||||
Select(state, state.uint_type, carry, ConstantU32(state, 1), ConstantU32(state, 0)));
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
uint32_t low, uint32_t high) {
|
||||
auto& state = ctx.state;
|
||||
const auto immediate = static_cast<int32_t>(mem.offset);
|
||||
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
|
||||
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
|
||||
const auto base = state.program.info.addresses[mem.resource].specialized_base;
|
||||
if (mem.kind == IR::ResourceKind::Scratch) {
|
||||
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
|
||||
const auto valid = Binary(state, OpIEqual, state.bool_type, high, ConstantU32(state, 0));
|
||||
return Select(state, state.uint_type, valid, low, ConstantU32(state, UINT32_MAX));
|
||||
}
|
||||
if (mem.kind == IR::ResourceKind::Flat ||
|
||||
(mem.kind == IR::ResourceKind::Global &&
|
||||
state.program.info.addresses[mem.resource].unbased)) {
|
||||
low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
|
||||
const auto base_low = ConstantU32(state, static_cast<uint32_t>(base));
|
||||
const auto base_high = ConstantU32(state, static_cast<uint32_t>(base >> 32u));
|
||||
const auto relative = Binary(state, OpISub, state.uint_type, low, base_low);
|
||||
const auto borrow = Binary(state, OpULessThan, state.bool_type, low, base_low);
|
||||
const auto relative_high = Binary(
|
||||
state, OpISub, state.uint_type, Binary(state, OpISub, state.uint_type, high, base_high),
|
||||
Select(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0)));
|
||||
const auto valid =
|
||||
Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0));
|
||||
return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX));
|
||||
}
|
||||
const auto initial = base + static_cast<uint64_t>(static_cast<int64_t>(immediate));
|
||||
auto relative_high = ConstantU32(state, static_cast<uint32_t>(initial >> 32u));
|
||||
const auto relative = AddU64Low(state, ConstantU32(state, static_cast<uint32_t>(initial)),
|
||||
relative_high, low, ConstantU32(state, 0), relative_high);
|
||||
const auto valid =
|
||||
Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0));
|
||||
return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX));
|
||||
}
|
||||
|
||||
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
if (mem.kind == IR::ResourceKind::Buffer || mem.kind == IR::ResourceKind::ScalarBuffer) {
|
||||
if (mem.kind == IR::ResourceKind::ScalarBuffer) {
|
||||
return Binary(ctx.state, OpIAdd, ctx.state.uint_type, ctx.Arg(inst, 1),
|
||||
ConstantU32(ctx.state, mem.offset));
|
||||
}
|
||||
return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2),
|
||||
ctx.Arg(inst, 3));
|
||||
}
|
||||
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
|
||||
return ctx.Arg(inst, 1);
|
||||
}
|
||||
return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
|
||||
}
|
||||
|
||||
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
return Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, ByteAddress(ctx, inst, mem),
|
||||
ConstantU32(ctx.state, 2));
|
||||
}
|
||||
|
||||
uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
|
||||
if (mem.kind == IR::ResourceKind::Lds) {
|
||||
return 0;
|
||||
}
|
||||
if (mem.kind == IR::ResourceKind::Gds) {
|
||||
return EmitGdsElementInBounds(ctx.state, index);
|
||||
}
|
||||
return EmitStorageBufferElementInBounds(ctx.state, mem, index, pc);
|
||||
}
|
||||
|
||||
uint32_t Pointer(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
|
||||
return EmitMemoryElementPointer(ctx.state, mem, index, pc);
|
||||
}
|
||||
|
||||
uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
|
||||
uint32_t extra_offset = 0) {
|
||||
mem.offset += extra_offset;
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
const auto value = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpLoad, ctx.state.uint_type, value, Pointer(ctx, mem, index, pc)});
|
||||
return value;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
|
||||
bool sign_extend, uint32_t extra_offset = 0) {
|
||||
mem.offset += extra_offset;
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address,
|
||||
ConstantU32(ctx.state, 2));
|
||||
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
const auto word = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpLoad, ctx.state.uint_type, word, Pointer(ctx, mem, index, pc)});
|
||||
const auto byte = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, byte,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto value =
|
||||
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type,
|
||||
Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, word, shift),
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
|
||||
if (!sign_extend) return value;
|
||||
const auto left = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, value,
|
||||
ConstantU32(ctx.state, 32u - bits));
|
||||
return Binary(ctx.state, OpShiftRightArithmetic, ctx.state.uint_type, left,
|
||||
ConstantU32(ctx.state, 32u - bits));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& inst,
|
||||
const IR::MemoryInfo& mem) {
|
||||
return mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
|
||||
: StorageBufferFormat(ctx.state, mem, inst.Flags<IR::MemoryFlags>().pc);
|
||||
}
|
||||
|
||||
IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format,
|
||||
uint32_t component) {
|
||||
if (!mem.typed && component != 0u) {
|
||||
const auto component_slot = component * 4u;
|
||||
mem.offset = mem.offset >= component_slot ? mem.offset - component_slot : 0u;
|
||||
mem.offset += Format::GetFormatComponentByteOffset(format, component);
|
||||
}
|
||||
return mem;
|
||||
}
|
||||
|
||||
uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
const auto format = BufferFormat(ctx, inst, mem);
|
||||
if (!Format::IsKnownFormat(format)) return LoadWord(ctx, inst, mem);
|
||||
const auto info = Format::GetFormatInfo(format);
|
||||
const auto component = mem.component_index;
|
||||
if (component >= info.component_count) return ConstantU32(ctx.state, 0);
|
||||
uint32_t raw = 0;
|
||||
const auto bits = info.component_bits[component];
|
||||
if (info.packed_bitfield) {
|
||||
raw = LoadWord(ctx, inst, mem);
|
||||
const auto type =
|
||||
IsSignedFormatComponent(info.type) ? ctx.state.int_type : ctx.state.uint_type;
|
||||
const auto source =
|
||||
type == ctx.state.int_type ? Unary(ctx.state, OpBitcast, type, raw) : raw;
|
||||
const auto extracted = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type,
|
||||
extracted, source, ConstantU32(ctx.state, info.component_bit_offset[component]),
|
||||
ConstantU32(ctx.state, bits)});
|
||||
raw = type == ctx.state.int_type
|
||||
? Unary(ctx.state, OpBitcast, ctx.state.uint_type, extracted)
|
||||
: extracted;
|
||||
} else if (bits == 32u) {
|
||||
raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component));
|
||||
} else {
|
||||
raw = LoadSubword(ctx, inst, RebaseFormattedComponent(mem, format, component), bits,
|
||||
IsSignedFormatComponent(info.type));
|
||||
}
|
||||
return NormalizeFormatComponent(ctx.state, info, component, raw);
|
||||
}
|
||||
|
||||
template <typename Fn>
|
||||
uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
|
||||
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
|
||||
const auto memory = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory
|
||||
: MemorySemanticsUniformMemory;
|
||||
const auto preheader = state.builder.AllocateId();
|
||||
const auto header = state.builder.AllocateId();
|
||||
const auto cont = state.builder.AllocateId();
|
||||
const auto merge = state.builder.AllocateId();
|
||||
const auto initial = state.builder.AllocateId();
|
||||
const auto observed = state.builder.AllocateId();
|
||||
const auto exchanged = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBranch, preheader});
|
||||
state.builder.AddFunction({OpLabel, preheader});
|
||||
state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
|
||||
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
|
||||
state.builder.AddFunction({OpBranch, header});
|
||||
state.builder.AddFunction({OpLabel, header});
|
||||
state.builder.AddFunction(
|
||||
{OpPhi, state.uint_type, observed, initial, preheader, exchanged, cont});
|
||||
const auto next = desired(observed);
|
||||
state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
|
||||
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
|
||||
ConstantU32(state, MemorySemanticsNone), next, observed});
|
||||
const auto success = Binary(state, OpIEqual, state.bool_type, exchanged, observed);
|
||||
state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone});
|
||||
state.builder.AddFunction({OpBranchConditional, success, merge, cont});
|
||||
state.builder.AddFunction({OpLabel, cont});
|
||||
state.builder.AddFunction({OpBranch, header});
|
||||
state.builder.AddFunction({OpLabel, merge});
|
||||
state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope),
|
||||
ConstantU32(state, MemorySemanticsAcquireRelease | memory)});
|
||||
return observed;
|
||||
}
|
||||
|
||||
void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
|
||||
uint32_t extra_offset = 0) {
|
||||
mem.offset += extra_offset;
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address,
|
||||
ConstantU32(ctx.state, 2));
|
||||
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
const auto pointer = Pointer(ctx, mem, index, pc);
|
||||
const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
|
||||
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address,
|
||||
ConstantU32(ctx.state, 3)),
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto mask = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
|
||||
const auto value = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
|
||||
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type,
|
||||
ctx.Arg(inst, inst.NumArgs() - 2),
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
|
||||
shift);
|
||||
const auto merge = [&](uint32_t old) {
|
||||
return Binary(ctx.state, OpBitwiseOr, ctx.state.uint_type,
|
||||
Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, old,
|
||||
Unary(ctx.state, OpNot, ctx.state.uint_type, mask)),
|
||||
value);
|
||||
};
|
||||
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
|
||||
AtomicUpdate(ctx.state, pointer, mem.kind, merge);
|
||||
} else {
|
||||
const auto old = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction({OpLoad, ctx.state.uint_type, old, pointer});
|
||||
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
|
||||
uint32_t extra_offset = 0) {
|
||||
mem.offset += extra_offset;
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpStore, Pointer(ctx, mem, index, pc), ctx.Arg(inst, inst.NumArgs() - 2)});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
const auto format = BufferFormat(ctx, inst, mem);
|
||||
if (!Format::IsKnownFormat(format)) {
|
||||
StoreWord(ctx, inst, mem);
|
||||
return;
|
||||
}
|
||||
const auto info = Format::GetFormatInfo(format);
|
||||
if (mem.component_index >= info.component_count) return;
|
||||
const auto bits = info.component_bits[mem.component_index];
|
||||
const auto component_mem = RebaseFormattedComponent(mem, format, mem.component_index);
|
||||
if (bits == 8u || bits == 16u) {
|
||||
StoreSubword(ctx, inst, component_mem, bits);
|
||||
} else {
|
||||
StoreWord(ctx, inst, component_mem);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
|
||||
switch (opcode) {
|
||||
case IR::ValueOpcode::BufferAtomicSwap32:
|
||||
case IR::ValueOpcode::SharedAtomicSwap32:
|
||||
case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange;
|
||||
case IR::ValueOpcode::BufferAtomicIAdd32:
|
||||
case IR::ValueOpcode::SharedAtomicIAdd32:
|
||||
case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd;
|
||||
case IR::ValueOpcode::BufferAtomicISub32:
|
||||
case IR::ValueOpcode::SharedAtomicISub32:
|
||||
case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub;
|
||||
case IR::ValueOpcode::BufferAtomicSMin32:
|
||||
case IR::ValueOpcode::SharedAtomicSMin32:
|
||||
case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin;
|
||||
case IR::ValueOpcode::BufferAtomicUMin32:
|
||||
case IR::ValueOpcode::SharedAtomicUMin32:
|
||||
case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin;
|
||||
case IR::ValueOpcode::BufferAtomicSMax32:
|
||||
case IR::ValueOpcode::SharedAtomicSMax32:
|
||||
case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax;
|
||||
case IR::ValueOpcode::BufferAtomicUMax32:
|
||||
case IR::ValueOpcode::SharedAtomicUMax32:
|
||||
case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax;
|
||||
case IR::ValueOpcode::BufferAtomicAnd32:
|
||||
case IR::ValueOpcode::SharedAtomicAnd32:
|
||||
case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd;
|
||||
case IR::ValueOpcode::BufferAtomicOr32:
|
||||
case IR::ValueOpcode::SharedAtomicOr32:
|
||||
case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr;
|
||||
case IR::ValueOpcode::BufferAtomicXor32:
|
||||
case IR::ValueOpcode::SharedAtomicXor32:
|
||||
case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
bool return_value) {
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
const auto result =
|
||||
EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
|
||||
const auto old = ctx.state.builder.AllocateId();
|
||||
const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
|
||||
ctx.state.builder.AddFunction({AtomicOpcode(inst.GetOpcode()), ctx.state.uint_type,
|
||||
old, Pointer(ctx, mem, index, pc),
|
||||
ConstantU32(ctx.state, scope),
|
||||
ConstantU32(ctx.state, MemorySemanticsNone), value});
|
||||
if (mem.kind == IR::ResourceKind::Lds) {
|
||||
const auto semantics =
|
||||
MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
|
||||
ctx.state.builder.AddFunction({OpMemoryBarrier,
|
||||
ConstantU32(ctx.state, ScopeWorkgroup),
|
||||
ConstantU32(ctx.state, semantics)});
|
||||
} else {
|
||||
EmitDeviceAtomicMemoryBarrier(ctx.state);
|
||||
}
|
||||
return old;
|
||||
});
|
||||
});
|
||||
return return_value ? result : ConstantU32(ctx.state, 0);
|
||||
}
|
||||
|
||||
uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
|
||||
bool max_value) {
|
||||
struct OrderedBits {
|
||||
uint32_t nan;
|
||||
uint32_t zero;
|
||||
uint32_t key;
|
||||
};
|
||||
const auto classify = [&](uint32_t bits) {
|
||||
const auto value = Unary(state, OpBitcast, state.float_type, bits);
|
||||
const auto cls = EmitClassifyF32(state, value);
|
||||
const auto negative = Binary(
|
||||
state, OpINotEqual, state.bool_type,
|
||||
Binary(state, OpBitwiseAnd, state.uint_type, bits, ConstantU32(state, 0x80000000u)),
|
||||
ConstantU32(state, 0));
|
||||
const auto negative_key = Unary(state, OpNot, state.uint_type, bits);
|
||||
const auto positive_key =
|
||||
Binary(state, OpBitwiseXor, state.uint_type, bits, ConstantU32(state, 0x80000000u));
|
||||
return OrderedBits {cls.nan, cls.zero,
|
||||
Select(state, state.uint_type, negative, negative_key, positive_key)};
|
||||
};
|
||||
const auto source_class = classify(source);
|
||||
const auto old_class = classify(old);
|
||||
const auto unordered =
|
||||
Binary(state, OpLogicalOr, state.bool_type,
|
||||
Binary(state, OpLogicalOr, state.bool_type, source_class.nan, old_class.nan),
|
||||
Binary(state, OpLogicalAnd, state.bool_type, source_class.zero, old_class.zero));
|
||||
const auto ordered = Unary(state, OpLogicalNot, state.bool_type, unordered);
|
||||
const auto compare = Binary(state, max_value ? OpUGreaterThan : OpULessThan, state.bool_type,
|
||||
source_class.key, old_class.key);
|
||||
return Select(state, state.uint_type,
|
||||
Binary(state, OpLogicalAnd, state.bool_type, ordered, compare), source, old);
|
||||
}
|
||||
|
||||
uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
bool max_value) {
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
const auto pointer = Pointer(ctx, mem, index, pc);
|
||||
const auto source = ctx.Arg(inst, inst.NumArgs() - 2);
|
||||
return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) {
|
||||
return FloatAtomicReplacement(ctx.state, old, source, max_value);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
bool max_value) {
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
|
||||
ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
|
||||
const auto data = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpLoad, ctx.state.uint_type, data, ctx.scratch_u32_variable});
|
||||
AtomicUpdate(ctx.state, Pointer(ctx, mem, index, pc), mem.kind, [&](uint32_t old) {
|
||||
const auto old_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, old);
|
||||
const auto compare_f =
|
||||
Unary(ctx.state, OpBitcast, ctx.state.float_type, ctx.Arg(inst, 3));
|
||||
const auto data_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, data);
|
||||
const auto compare = Binary(
|
||||
ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, ctx.state.bool_type,
|
||||
max_value ? old_f : compare_f, max_value ? compare_f : old_f);
|
||||
return Unary(ctx.state, OpBitcast, ctx.state.uint_type,
|
||||
Select(ctx.state, ctx.state.float_type, compare, data_f, old_f));
|
||||
});
|
||||
});
|
||||
});
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) {
|
||||
auto& state = ctx.state;
|
||||
const auto m0 = ctx.Arg(inst, 1);
|
||||
const auto base =
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, m0, ConstantU32(state, 16));
|
||||
const auto size = Binary(state, OpBitwiseAnd, state.uint_type, m0, ConstantU32(state, 0xffffu));
|
||||
const auto address =
|
||||
Binary(state, OpIAdd, state.uint_type, base, ConstantU32(state, ctx.Memory(inst).offset));
|
||||
const auto index =
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2));
|
||||
const auto exec = ctx.Arg(inst, 2);
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), exec});
|
||||
const auto low = state.builder.AllocateId();
|
||||
const auto high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
|
||||
const auto count_low = state.per_invocation_masks ? low : ctx.Arg(inst, 3);
|
||||
const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4);
|
||||
const auto count =
|
||||
Binary(state, OpIAdd, state.uint_type, Unary(state, OpBitCount, state.uint_type, count_low),
|
||||
Unary(state, OpBitCount, state.uint_type, count_high));
|
||||
const auto first = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first,
|
||||
ConstantU32(state, ScopeSubgroup), ballot});
|
||||
const auto is_first =
|
||||
Binary(state, OpIEqual, state.bool_type, EmitSubgroupLocalInvocationId(state), first);
|
||||
const auto storage_bounds = gds ? EmitGdsElementInBounds(state, index) : 0u;
|
||||
const auto m0_bounds =
|
||||
gds ? Binary(state, OpINotEqual, state.bool_type, size, ConstantU32(state, 0))
|
||||
: Binary(state, OpULessThan, state.bool_type,
|
||||
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
|
||||
const auto condition = AndCondition(
|
||||
state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
|
||||
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{append ? OpAtomicIAdd : OpAtomicISub, state.uint_type, value,
|
||||
gds ? EmitGdsElementPointer(state, index) : EmitLdsElementPointer(state, index),
|
||||
ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
|
||||
ConstantU32(state, MemorySemanticsNone), count});
|
||||
return value;
|
||||
});
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, result,
|
||||
ConstantU32(state, ScopeSubgroup), atomic, first});
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
auto& state = ctx.state;
|
||||
const auto op = inst.GetOpcode();
|
||||
if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
|
||||
ctx.Memory(inst).planning_only) {
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::ReadConst) {
|
||||
if (state.flattened_srt_variable == 0) {
|
||||
ctx.Fail(inst, "requires the flattened SRT descriptor");
|
||||
return true;
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
|
||||
state.flattened_srt_variable, ConstantU32(state, 0),
|
||||
ctx.Arg(inst, 1)});
|
||||
ctx.Emit(inst, OpLoad, IR::Type::U32, {pointer});
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::ReadConstBuffer) {
|
||||
auto mem = ctx.Memory(inst);
|
||||
mem.kind = IR::ResourceKind::ScalarBuffer;
|
||||
const auto address = Binary(state, OpIAdd, state.uint_type, ctx.Arg(inst, 1),
|
||||
ConstantU32(state, mem.offset));
|
||||
const auto index =
|
||||
Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2));
|
||||
const auto pc = inst.Flags<IR::MemoryFlags>().pc;
|
||||
const auto condition = EmitStorageBufferElementInBounds(state, mem, index, pc);
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, value,
|
||||
EmitStorageBufferElementPointer(state, mem, index, pc)});
|
||||
return value;
|
||||
}));
|
||||
return true;
|
||||
}
|
||||
const bool load_address = op == IR::ValueOpcode::LoadAddressU8 ||
|
||||
op == IR::ValueOpcode::LoadAddressU16 ||
|
||||
op == IR::ValueOpcode::LoadAddressU32;
|
||||
const bool load_buffer = op == IR::ValueOpcode::LoadBufferU8 ||
|
||||
op == IR::ValueOpcode::LoadBufferU16 ||
|
||||
op == IR::ValueOpcode::LoadBufferU32;
|
||||
const bool load_shared =
|
||||
op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 ||
|
||||
op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 ||
|
||||
op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32;
|
||||
if (load_address || load_buffer || load_shared) {
|
||||
auto mem = ctx.Memory(inst);
|
||||
if (load_shared)
|
||||
mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 ||
|
||||
op == IR::ValueOpcode::LoadGdsU32
|
||||
? IR::ResourceKind::Gds
|
||||
: IR::ResourceKind::Lds;
|
||||
uint32_t value = 0;
|
||||
if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
|
||||
value = FormattedLoad(ctx, inst, mem);
|
||||
else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 ||
|
||||
op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8)
|
||||
value = LoadSubword(ctx, inst, mem, 8, false);
|
||||
else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 ||
|
||||
op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16)
|
||||
value = LoadSubword(ctx, inst, mem, 16, false);
|
||||
else
|
||||
value = LoadWord(ctx, inst, mem);
|
||||
ctx.Define(inst, value);
|
||||
return true;
|
||||
}
|
||||
const bool store_address = op == IR::ValueOpcode::StoreAddressU8 ||
|
||||
op == IR::ValueOpcode::StoreAddressU16 ||
|
||||
op == IR::ValueOpcode::StoreAddressU32;
|
||||
const bool store_buffer = op == IR::ValueOpcode::StoreBufferU8 ||
|
||||
op == IR::ValueOpcode::StoreBufferU16 ||
|
||||
op == IR::ValueOpcode::StoreBufferU32;
|
||||
const bool store_shared =
|
||||
op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 ||
|
||||
op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 ||
|
||||
op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32;
|
||||
if (store_address || store_buffer || store_shared) {
|
||||
auto mem = ctx.Memory(inst);
|
||||
if (store_shared)
|
||||
mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 ||
|
||||
op == IR::ValueOpcode::WriteGdsU32
|
||||
? IR::ResourceKind::Gds
|
||||
: IR::ResourceKind::Lds;
|
||||
if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
|
||||
FormattedStore(ctx, inst, mem);
|
||||
else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
|
||||
op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8)
|
||||
StoreSubword(ctx, inst, mem, 8);
|
||||
else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
|
||||
op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16)
|
||||
StoreSubword(ctx, inst, mem, 16);
|
||||
else
|
||||
StoreWord(ctx, inst, mem);
|
||||
return true;
|
||||
}
|
||||
const auto atomic_opcode = AtomicOpcode(op);
|
||||
if (atomic_opcode != 0) {
|
||||
auto mem = ctx.Memory(inst);
|
||||
if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32)
|
||||
mem.kind = IR::ResourceKind::Lds;
|
||||
else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32)
|
||||
mem.kind = IR::ResourceKind::Gds;
|
||||
ctx.Define(inst, EmitAtomic(ctx, inst, mem, true));
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
|
||||
ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst),
|
||||
op == IR::ValueOpcode::BufferAtomicFMax32));
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 ||
|
||||
op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) {
|
||||
auto mem = ctx.Memory(inst);
|
||||
mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32
|
||||
? IR::ResourceKind::Gds
|
||||
: IR::ResourceKind::Lds;
|
||||
SharedFloatAtomic(ctx, inst, mem,
|
||||
op == IR::ValueOpcode::SharedAtomicFMax32 ||
|
||||
op == IR::ValueOpcode::GdsAtomicFMax32);
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume ||
|
||||
op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) {
|
||||
ctx.Define(inst, AppendConsume(ctx, inst,
|
||||
op == IR::ValueOpcode::GdsDataAppend ||
|
||||
op == IR::ValueOpcode::GdsDataConsume,
|
||||
op == IR::ValueOpcode::DataAppend ||
|
||||
op == IR::ValueOpcode::GdsDataAppend));
|
||||
return true;
|
||||
}
|
||||
if (op == IR::ValueOpcode::SwizzleU32) {
|
||||
ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)});
|
||||
const auto source = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
{OpLoad, ctx.state.uint_type, source, ctx.scratch_u32_variable});
|
||||
const auto target =
|
||||
EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
|
||||
inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), source, target});
|
||||
const auto source_exec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
|
||||
ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 2), target});
|
||||
const auto source_active =
|
||||
AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target));
|
||||
ctx.Define(inst,
|
||||
Select(state, state.uint_type, source_active, shuffled, ConstantU32(state, 0)));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,839 @@
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
|
||||
if (sel <= 3u) {
|
||||
offset = sel * 8u;
|
||||
width = 8u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 4u || sel == 5u) {
|
||||
offset = sel == 5u ? 16u : 0u;
|
||||
width = 16u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 6u) {
|
||||
offset = 0;
|
||||
width = 32u;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
|
||||
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t offset = 0;
|
||||
uint32_t width = 0;
|
||||
if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (operand.sdwa_sext) {
|
||||
const auto signed_value = state.builder.AllocateId();
|
||||
const auto extracted = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value});
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted});
|
||||
return bits;
|
||||
}
|
||||
|
||||
const auto extracted = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
return extracted;
|
||||
}
|
||||
|
||||
uint32_t EmitTrueBool(EmitterState& state) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
|
||||
const auto quad_base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto selected0 = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
|
||||
bool left) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane_shifted = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto valid = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
if (left) {
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
|
||||
}
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
|
||||
return {target, valid};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
|
||||
uint32_t amount) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto minus = state.builder.AllocateId();
|
||||
const auto plus = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
|
||||
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
|
||||
const auto lane_mask = half_row ? 7u : 15u;
|
||||
const auto base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto mirrored = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
if (control <= 0xffu) {
|
||||
return EmitDppQuadPermTargetLane(state, subid, control);
|
||||
}
|
||||
if (control >= 0x101u && control <= 0x10fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
|
||||
}
|
||||
if (control >= 0x111u && control <= 0x11fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
|
||||
}
|
||||
if (control >= 0x121u && control <= 0x12fu) {
|
||||
return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
|
||||
}
|
||||
if (control == 0x140u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, false);
|
||||
}
|
||||
if (control == 0x141u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, true);
|
||||
}
|
||||
return {subid, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
|
||||
if (!operand.dpp) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target.lane});
|
||||
if (operand.dpp_fetch_inactive) {
|
||||
return shuffled;
|
||||
}
|
||||
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
|
||||
const auto can_fetch = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
|
||||
uint32_t value = 0;
|
||||
switch (operand.kind) {
|
||||
case IR::OperandKind::ImmediateU32:
|
||||
case IR::OperandKind::PcRelativeU32:
|
||||
case IR::OperandKind::PcRelativeHighU32: value = ConstantU32(state, operand.imm); break;
|
||||
case IR::OperandKind::Register: {
|
||||
const auto pointer = PointerForRegister(state, operand.reg);
|
||||
if (pointer == 0) {
|
||||
value = ConstantU32(state, 0);
|
||||
break;
|
||||
}
|
||||
value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
break;
|
||||
}
|
||||
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
|
||||
default: value = ConstantU32(state, 0); break;
|
||||
}
|
||||
value = EmitDppValueU32(state, operand, value);
|
||||
return EmitSdwaExtractU32(state, operand, value);
|
||||
}
|
||||
|
||||
uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) {
|
||||
if (IsInactiveWave32ExecHigh(state, reg)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = PointerForRegister(state, reg);
|
||||
if (pointer == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) {
|
||||
if (state.wave_size == 32u && IsMaskRegisterFile(file)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitRegisterLoad(state, {file, 1});
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
|
||||
if (part == 0) {
|
||||
const auto in_low = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
if (state.wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto below = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)});
|
||||
|
||||
if (part == 0) {
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
|
||||
return ret;
|
||||
}
|
||||
if (state.wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto condition = state.builder.AllocateId();
|
||||
if (state.per_invocation_masks) {
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
if (state.exact_subgroup_operations) {
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto low_hit = state.builder.AllocateId();
|
||||
const auto high_hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
|
||||
if (state.per_invocation_masks) {
|
||||
return EmitMaskZeroBool(state, file, zero);
|
||||
}
|
||||
if (state.exact_subgroup_operations) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto combined = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
|
||||
uint32_t EmitExecActiveBool(EmitterState& state) {
|
||||
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
|
||||
}
|
||||
|
||||
uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) {
|
||||
if (operand.kind == IR::OperandKind::Register) {
|
||||
if (operand.reg.file == IR::RegisterFile::Exec ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc) {
|
||||
return EmitMaskActiveBool(state, operand.reg.file);
|
||||
}
|
||||
}
|
||||
const auto cond_value = EmitValueLoad(state, operand);
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
|
||||
if (state.subgroup_local_invocation_id_variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == kind) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return &input;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
|
||||
const auto variable = InputVariableForKind(state, kind);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t EmitLocalInvocationIndex(EmitterState& state) {
|
||||
const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t component) {
|
||||
if ((component & 3u) != 3u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
|
||||
}
|
||||
|
||||
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
|
||||
uint32_t component) {
|
||||
const auto count = VertexParameterComponentCount(state, input);
|
||||
const auto kind = VertexParameterScalarKind(state, input.location);
|
||||
if (component >= count) {
|
||||
return VertexInputDefaultComponentU32(state, kind, component);
|
||||
}
|
||||
|
||||
const auto scalar_type = VertexParameterScalarType(state, kind);
|
||||
uint32_t raw = state.builder.AllocateId();
|
||||
if (count == 1u) {
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
|
||||
} else {
|
||||
const auto pointer_type = VertexParameterScalarPointerType(state, kind);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
|
||||
ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
|
||||
}
|
||||
|
||||
if (kind == VertexInputScalarKind::Uint) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
|
||||
return bits;
|
||||
}
|
||||
|
||||
void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto* input = InputBindingForParameter(state, inst.input_info.attr);
|
||||
if (input == nullptr || input->variable_id == 0) {
|
||||
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.stage == ShaderType::Vertex) {
|
||||
EmitStoreU32(state, inst.dst,
|
||||
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
|
||||
return;
|
||||
}
|
||||
|
||||
const auto input_value = state.builder.AllocateId();
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t EmitSccBool(EmitterState& state, bool non_zero) {
|
||||
const auto value = EmitRegisterLoad(state, SccRegister());
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto uint_value = EmitValueLoad(state, operand);
|
||||
auto float_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value});
|
||||
float_value = abs_value;
|
||||
}
|
||||
if (operand.negate) {
|
||||
const auto neg_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value});
|
||||
float_value = neg_value;
|
||||
}
|
||||
return float_value;
|
||||
}
|
||||
|
||||
uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) {
|
||||
uint32_t result = value;
|
||||
if (dst.omod != 0) {
|
||||
uint32_t bits = 0x3f000000u;
|
||||
switch (dst.omod) {
|
||||
case 1u: bits = 0x40000000u; break;
|
||||
case 2u: bits = 0x40800000u; break;
|
||||
default: break;
|
||||
}
|
||||
const auto scaled = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)});
|
||||
result = scaled;
|
||||
}
|
||||
if (!dst.clamp) {
|
||||
return result;
|
||||
}
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp,
|
||||
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) {
|
||||
auto value_operand = operand;
|
||||
value_operand.negate_hi = false;
|
||||
|
||||
if (operand.op_sel_hi) {
|
||||
auto raw = operand;
|
||||
raw.sdwa_sel = 6;
|
||||
raw.sdwa_sext = false;
|
||||
raw.op_sel = false;
|
||||
raw.op_sel_hi = false;
|
||||
raw.negate = false;
|
||||
raw.negate_hi = false;
|
||||
raw.absolute = false;
|
||||
raw.dpp = false;
|
||||
raw.dpp_fetch_inactive = false;
|
||||
raw.dpp_bound_ctrl = false;
|
||||
|
||||
const auto bits = EmitValueLoad(state, raw);
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450,
|
||||
GlslUnpackHalf2x16, bits});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32});
|
||||
f32 = abs_value;
|
||||
}
|
||||
if (value_operand.negate) {
|
||||
const auto neg_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32});
|
||||
f32 = neg_value;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
|
||||
value_operand.op_sel = false;
|
||||
value_operand.op_sel_hi = false;
|
||||
return EmitFloatLoad(state, value_operand);
|
||||
}
|
||||
|
||||
IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind != IR::OperandKind::Register) {
|
||||
return operand;
|
||||
}
|
||||
auto ret = operand;
|
||||
ret.reg.index += offset;
|
||||
ret.sdwa_sel = 6;
|
||||
ret.omod = 0;
|
||||
ret.sdwa_sext = false;
|
||||
ret.op_sel = false;
|
||||
ret.op_sel_hi = false;
|
||||
ret.negate = false;
|
||||
ret.negate_hi = false;
|
||||
ret.absolute = false;
|
||||
ret.clamp = false;
|
||||
ret.dpp_ctrl = 0;
|
||||
ret.dpp_row_mask = 0xf;
|
||||
ret.dpp_bank_mask = 0xf;
|
||||
ret.dpp_fetch_inactive = false;
|
||||
ret.dpp_bound_ctrl = false;
|
||||
ret.dpp = false;
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto low = EmitValueLoad(state, operand);
|
||||
const auto active = state.builder.AllocateId();
|
||||
if (state.per_invocation_masks || state.exact_subgroup_operations) {
|
||||
uint32_t high = ConstantU32(state, 0);
|
||||
if (operand.kind == IR::OperandKind::Register &&
|
||||
(operand.reg.file == IR::RegisterFile::Scalar ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc ||
|
||||
operand.reg.file == IR::RegisterFile::Exec)) {
|
||||
high = IsMaskRegisterFile(operand.reg.file)
|
||||
? WaveMaskHighLoad(state, operand.reg.file)
|
||||
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
|
||||
}
|
||||
if (state.per_invocation_masks) {
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
const auto low_hit = state.builder.AllocateId();
|
||||
const auto high_hit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) {
|
||||
if (state.per_invocation_masks) {
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, ballot, lane);
|
||||
}
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
|
||||
const auto active_ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, active_ballot, lane);
|
||||
}
|
||||
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
|
||||
const auto low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
|
||||
uint32_t mask = low;
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
const auto in_range = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
|
||||
}
|
||||
if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) {
|
||||
return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u);
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
|
||||
const auto value = EmitSequentialValueLoad(state, operand, offset);
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
bool ImageAddressUsesA16(const IR::Instruction& inst) {
|
||||
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u;
|
||||
}
|
||||
|
||||
bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t cursor = 0;
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) {
|
||||
if (component == cursor) {
|
||||
return false;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) {
|
||||
if (component == cursor) {
|
||||
return true;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u &&
|
||||
component == cursor) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return component * 2u;
|
||||
}
|
||||
|
||||
uint32_t half_component = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u;
|
||||
if (half_width == 2u && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
half_component += half_width;
|
||||
}
|
||||
if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
return half_component;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
return ImageAddressHalfComponent(inst, component) / 2u;
|
||||
}
|
||||
|
||||
IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
|
||||
uint32_t component) {
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) {
|
||||
const auto nsa_index = packed_component - 1u;
|
||||
const auto nsa_count = std::min<uint32_t>(inst.memory.image_nsa_dwords * 4u,
|
||||
Decoder::MaxImageNsaAddressComponents);
|
||||
if (nsa_index < nsa_count) {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Vector,
|
||||
inst.memory.image_nsa_addr[nsa_index]);
|
||||
}
|
||||
}
|
||||
return OffsetRegisterOperand(base, packed_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto operand = ImageAddressOperand(inst, base, component);
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
auto value = EmitValueLoad(state, operand);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) {
|
||||
value = EmitShiftRightConstant(state, value, 16u);
|
||||
}
|
||||
value = EmitAndConstant(state, value, 0xffffu);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
return EmitF16BitsToF32(state, value);
|
||||
}
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
uint32_t EmitZeroF32(EmitterState& state) {
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)});
|
||||
return f32;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
Reference in New Issue
Block a user