raphiaura/KytyPS5
1
0
Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-21 11:26:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

shader: collect usage once and emit descriptor slots directly

Browse Source
This commit is contained in:
nmzik
2026-10-09 02:17:38 +04:00
parent b35cdc39b7
commit e1ac893a0c
28 changed files with 1068 additions and 1408 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp
+101 -242
View File
@@ -3,7 +3,6 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include <algorithm>
#include <array>
@@ -19,92 +18,14 @@ namespace {
}
void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
using Kind = IR::DescriptorBindingKind;
constexpr auto KindCount = static_cast<size_t>(Kind::Count);
std::array<std::vector<uint32_t>, KindCount> expected;
std::array<bool, KindCount> present {};
const auto Dense = [](size_t size) {
std::vector<uint32_t> values(size);
for (uint32_t i = 0; i < values.size(); i++) {
values[i] = i;
}
return values;
uint64_t live_buffers = 0;
bool uses_lds = false;
bool uses_gds = false;
const auto uses_mapping = [](const auto& resource) {
return resource.indirect_root != UINT32_MAX;
};
auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
const auto index = static_cast<size_t>(kind);
present[index] = true;
expected[index] = std::move(resources);
};
for (uint32_t i = 0; i < program.info.images.size(); i++) {
const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
if (!kind.has_value()) {
Fail(program, "native shader plan has an invalid image class");
}
present[static_cast<size_t>(*kind)] = true;
const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::Dynamic;
const auto count = dynamic ? program.info.images[i].mip_count : 1u;
if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
Fail(program, "native shader plan has an invalid image mip descriptor count");
}
expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(),
count, i);
}
if (!program.info.samplers.empty()) {
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
}
auto& buffers = expected[static_cast<size_t>(Kind::Buffers)];
const auto shared = IR::CollectMemoryResources(program, buffers);
present[static_cast<size_t>(Kind::Buffers)] = !buffers.empty();
if (shared.gds) {
Expect(Kind::Gds);
}
if (shared.lds && lds_storage) {
Expect(Kind::SharedMemory);
}
if (program.info.uses_dma) {
Expect(Kind::BdaPagetable);
Expect(Kind::FaultBuffer);
}
if (IR::UsesFlattenedSrt(program)) {
Expect(Kind::FlattenedSrt);
}
if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) {
Expect(Kind::ShaderData);
}
std::array<bool, KindCount> seen {};
for (const auto& binding: program.bindings.descriptors) {
const auto kind = static_cast<size_t>(binding.kind);
if (kind >= KindCount || seen[kind] || !present[kind] ||
binding.resources != expected[kind]) {
Fail(program, "native descriptor groups do not match shader topology");
}
seen[kind] = true;
}
for (size_t i = 0; i < KindCount; i++) {
if (present[i] != seen[i]) {
Fail(program, "native shader plan is missing a required descriptor group");
}
}
const auto has_shader_data_storage = present[static_cast<size_t>(Kind::ShaderData)];
const auto shader_data_dwords = program.bindings.ShaderDataDwords();
const auto user_data_dwords = program.bindings.user_data_registers.size();
const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart;
if ((program.bindings.UsesPushData() &&
!IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
(has_dispatch_threads && (program.stage != ShaderType::Compute ||
program.bindings.dispatch_thread_dword != user_data_dwords)) ||
program.bindings.memory_offset_dword != user_data_dwords + (has_dispatch_threads ? 3u : 0u) ||
program.bindings.memory_offset_count != buffers.size() ||
has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) ||
!std::is_sorted(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) ||
std::adjacent_find(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) !=
program.bindings.user_data_registers.end()) {
Fail(program, "native shader-data layout is inconsistent");
}
bool uses_flattened_srt = std::ranges::any_of(program.info.buffers, uses_mapping) ||
std::ranges::any_of(program.info.images, uses_mapping);
const auto planning_only_handle = [&](const IR::Inst& handle) {
return !handle.Uses().empty() &&
std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
@@ -141,6 +62,39 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
};
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
const auto op = inst.GetOpcode();
uses_flattened_srt |= op == IR::ValueOpcode::ReadConst;
if (IR::BufferAccessOf(op) != IR::BufferAccess::None ||
IR::SharedAccessOf(op) != IR::SharedAccess::None ||
IR::AddressOpcodeInfoOf(op).access != IR::AddressAccess::None) {
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.memory_info.size()) {
Fail(program, "typed shader contains invalid memory metadata");
}
const auto& memory = program.memory_info[index];
if (!memory.planning_only) {
uses_lds |= memory.kind == IR::ResourceKind::FlatLocal;
if (IR::SharedAccessOf(op) != IR::SharedAccess::None) {
if (memory.kind != IR::ResourceKind::Lds && memory.kind != IR::ResourceKind::Gds) {
Fail(program, "typed shader contains invalid shared-memory metadata");
}
uses_gds |= memory.kind == IR::ResourceKind::Gds;
uses_lds |= memory.kind == IR::ResourceKind::Lds;
} else if (memory.kind == IR::ResourceKind::Buffer ||
memory.kind == IR::ResourceKind::ScalarBuffer) {
if (memory.resource >= program.info.buffers.size()) {
Fail(program, "typed shader contains an invalid buffer resource");
}
live_buffers |= uint64_t{1} << memory.resource;
for (const auto child: program.info.buffers[memory.resource].indirect_resources) {
if (child >= program.info.buffers.size()) {
Fail(program, "typed shader contains an invalid indirect buffer resource");
}
live_buffers |= uint64_t{1} << child;
}
}
}
}
const auto dense = inst.Flags<uint32_t>();
switch (inst.GetOpcode()) {
case IR::ValueOpcode::GetBufferResource:
@@ -186,163 +140,64 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
}
}
}
using Kind = IR::DescriptorBindingKind;
std::array<uint32_t, static_cast<size_t>(Kind::Count)> expected {};
auto& buffer_count = expected[static_cast<size_t>(Kind::Buffers)];
for (uint32_t index = 0; index < program.info.buffers.size(); ++index) {
const auto slot = (live_buffers & (uint64_t{1} << index)) != 0 ? buffer_count++ : UINT32_MAX;
if (program.info.buffers[index].descriptor_index != slot) {
Fail(program, "native buffer descriptor index does not match shader topology");
}
}
for (const auto& image: program.info.images) {
const auto kind = IR::DescriptorBindingForImage(image);
if (!kind.has_value()) {
Fail(program, "native shader plan has an invalid image class");
}
if (image.mip_count == 0u ||
(image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
Fail(program, "native shader plan has an invalid image mip descriptor count");
}
auto& count = expected[static_cast<size_t>(*kind)];
if (image.descriptor_index != count) {
Fail(program, "native image descriptor index does not match shader topology");
}
count += image.mip_count;
}
expected[static_cast<size_t>(Kind::Samplers)] = static_cast<uint32_t>(program.info.samplers.size());
expected[static_cast<size_t>(Kind::Gds)] = uses_gds;
expected[static_cast<size_t>(Kind::SharedMemory)] = uses_lds && lds_storage;
expected[static_cast<size_t>(Kind::BdaPagetable)] = program.info.uses_dma;
expected[static_cast<size_t>(Kind::FaultBuffer)] = program.info.uses_dma;
expected[static_cast<size_t>(Kind::FlattenedSrt)] = uses_flattened_srt;
expected[static_cast<size_t>(Kind::ShaderData)] =
program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData();
uint64_t descriptor_mask = 0;
for (uint32_t index = 0; index < expected.size(); ++index) {
if (expected[index] != 0) descriptor_mask |= uint64_t{1} << index;
}
if (program.bindings.descriptor_counts != expected ||
program.bindings.descriptor_mask != descriptor_mask) {
Fail(program, "native descriptor counts do not match shader topology");
}
const auto shader_data_dwords = program.bindings.ShaderDataDwords();
const auto& registers = program.info.user_data_registers;
const bool has_dispatch_threads = program.bindings.dispatch_thread_dword != IR::PushData::NoStart;
if ((program.bindings.UsesPushData() &&
!IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
(has_dispatch_threads && (program.stage != ShaderType::Compute ||
program.bindings.dispatch_thread_dword != registers.size())) ||
program.bindings.memory_offset_dword != registers.size() + (has_dispatch_threads ? 3u : 0u) ||
!std::is_sorted(registers.begin(), registers.end()) ||
std::adjacent_find(registers.begin(), registers.end()) != registers.end()) {
Fail(program, "native shader-data layout is inconsistent");
}
}
} // namespace
Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) {
SpirvRequirements requirements {};
for (const auto* block: program.blocks) {
for (const auto& inst: *block) {
requirements.float64 |= inst.GetType() == IR::Type::F64;
if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
inst.GetType() == IR::Type::U64) {
requirements.buffer_int64_atomics = true;
}
const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
if (address_access != IR::AddressAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
Fail(program, "address operation has invalid memory metadata");
}
const auto& memory = program.memory_info[memory_index];
const auto kind = memory.kind;
if (kind == IR::ResourceKind::Scratch || kind == IR::ResourceKind::FlatLocal) {
if (program.scratch_dwords == 0) {
Fail(program, "scratch operation has no per-thread storage");
}
requirements.function_scratch = true;
if (kind == IR::ResourceKind::FlatLocal && program.stage != ShaderType::Compute &&
program.stage != ShaderType::Mesh) {
requirements.function_lds = true;
}
} else if (address_access == IR::AddressAccess::Write) {
Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
} else {
// Descriptor stores can alias coherent physical-address loads.
requirements.coherent_buffers |= memory.coherent;
}
}
if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
if (memory_index >= program.memory_info.size()) {
Fail(program, "buffer operation has invalid memory metadata");
}
const auto& memory = program.memory_info[memory_index];
if (memory.kind == IR::ResourceKind::IndirectBuffer &&
inst.GetOpcode() != IR::ValueOpcode::ReadConstBuffer) {
requirements.subgroup_local_invocation_id = true;
}
if (memory.kind == IR::ResourceKind::Buffer) {
requirements.coherent_buffers |= memory.coherent;
if (memory.resource >= program.info.buffers.size()) {
Fail(program, "buffer operation has invalid resource metadata");
}
const auto bits = StorageBufferElementBits(program, memory);
requirements.buffer_u8 |= bits == 8u;
requirements.buffer_u16 |= bits == 16u;
if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
if (program.stage != ShaderType::Compute) {
Fail(program, "buffer ADD_TID is only valid for compute shaders");
}
requirements.subgroup_local_invocation_id = true;
}
}
}
const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
if (shared_access != IR::SharedAccess::None) {
const auto index = inst.Flags<IR::MemoryFlags>().index;
if (index >= program.memory_info.size()) {
Fail(program, "shared operation has invalid memory metadata");
}
const auto kind = program.memory_info[index].kind;
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
Fail(program, "shared operation has invalid resource kind");
}
if (shared_access == IR::SharedAccess::Atomic &&
IR::SharedComponentCount(inst.GetOpcode()) == 2u) {
if (kind != IR::ResourceKind::Lds || program.stage != ShaderType::Compute) {
Fail(program, "64-bit shared atomics require compute LDS");
}
requirements.shared_int64_atomics = true;
}
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh &&
kind == IR::ResourceKind::Lds) {
requirements.function_lds = true;
}
if (shared_access == IR::SharedAccess::Append ||
shared_access == IR::SharedAccess::Consume) {
requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
}
}
switch (inst.GetOpcode()) {
case IR::ValueOpcode::StoreCompletion: requirements.subgroup_barrier = true; break;
case IR::ValueOpcode::BvhIntersect: requirements.bvh = true; break;
case IR::ValueOpcode::ConditionRef:
requirements.subgroup_ballot |=
inst.Flags<CFG::BranchCondition>() != CFG::BranchCondition::ScalarInstruction;
break;
case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::ReadFirstLane:
case IR::ValueOpcode::ReadLane: {
requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true;
if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
requirements.subgroup_local_invocation_id = true;
}
break;
}
case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WriteLane: {
requirements.subgroup_ballot = true;
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::Permlane16U32: {
requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::SwizzleU32:
case IR::ValueOpcode::PermuteU32:
case IR::ValueOpcode::BpermuteU32: {
requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true;
break;
}
case IR::ValueOpcode::LaneId:
requirements.subgroup_local_invocation_id |=
program.stage != ShaderType::TessellationControl;
break;
case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
case IR::ValueOpcode::ImageGatherRaw:
requirements.image_gather_extended = true;
break;
case IR::ValueOpcode::SetAttribute: {
const auto index = inst.Flags<IR::ExportFlags>().index;
if (index >= program.export_info.size()) {
Fail(program, "attribute export has invalid metadata");
}
if (program.stage == ShaderType::Pixel &&
program.export_info[index].vm) {
requirements.pixel_valid_mask = true;
}
break;
}
default: break;
}
}
}
return requirements;
}
std::vector<uint32_t> EmitProgram(const IR::Program& program,
ShaderStageInputInfo input_info) {
std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo input_info,
uint32_t push_data_start_dword) {
using namespace Emitter;
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
@@ -352,12 +207,14 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
Fail(program, "binary SPIR-V emitter received an unsupported shader stage");
}
if (!program.srt_plan_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete) {
!program.shader_info_complete) {
Fail(program, "SPIR-V emitter requires a fully planned native shader program");
}
ValidateNativeProgram(program, program.stage == ShaderType::Compute &&
input_info.compute != nullptr && input_info.compute->lds_storage);
if (program.info.buffers.size() > IR::ShaderInfo::MaxBuffers) {
Fail(program, "native shader exceeds the buffer resource limit");
}
IR::ValidateProgram(program, true);
program.bindings = {.push_data_start_dword = push_data_start_dword};
EmitterState state(program, input_info);
const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
state.lane_count =
@@ -365,6 +222,8 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
? 2u
: 1u;
DefineModule(state);
ValidateNativeProgram(program, program.stage == ShaderType::Compute &&
input_info.compute != nullptr && input_info.compute->lds_storage);
EmitProgram(state);
state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func,
"main", state.interface_variables);
src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h
+3 -3
View File
@@ -9,9 +9,9 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv {
// Resets and writes IR backend IDs; do not emit the same program concurrently.
std::vector<uint32_t> EmitProgram(const IR::Program& program,
ShaderStageInputInfo input_info);
// Resets and writes backend IDs and descriptor indices; do not emit the same program concurrently.
std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo input_info,
uint32_t push_data_start_dword = 0);
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp
+6 -19
View File
@@ -59,21 +59,6 @@ uint32_t ConstantU32(EmitterState& state, uint32_t value);
std::abort();
}
uint32_t ResourceForDescriptor(const EmitterState& state, IR::DescriptorBindingKind kind,
uint32_t resource) {
const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
if (descriptor == nullptr) {
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
}
const auto found =
std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
if (found == descriptor->resources.end()) {
ExitDescriptorBindingFailure(state, kind, resource,
"resource is absent from descriptor group");
}
return static_cast<uint32_t>(found - descriptor->resources.begin());
}
uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
@@ -162,13 +147,15 @@ uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mi
}
uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
const auto array_index =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
if (sampler >= state.program.info.samplers.size()) {
ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Samplers, sampler,
"sampler resource index is out of range");
}
const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
const auto pointer_type =
state.builder.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, sampler_type);
const auto pointer = DescriptorElementPointer(
state, pointer_type, state.sampler_variable, ConstantU32(state, array_index),
state, pointer_type, state.sampler_variable, ConstantU32(state, sampler),
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
const auto sampler_id = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, sampler_type, sampler_id, pointer);
@@ -200,7 +187,7 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t
const auto kind = IR::DescriptorBindingForImage(image);
EXIT_IF(!kind.has_value());
if (array_index == 0u) {
array_index = ConstantU32(state, ResourceForDescriptor(state, *kind, resource) + mip);
array_index = ConstantU32(state, image.descriptor_index + mip);
}
const auto pointer_type = state.builder.Type(
spv::OpTypePointer, spv::StorageClassUniformConstant, ImageType(state, image));
src/graphics/shader/recompiler/backend/spirv/spirvEmitterBvh.cpp
+1 -1
View File
@@ -223,7 +223,7 @@ uint32_t Boxes(EmitterState& s, uint32_t first, uint32_t second, uint32_t half,
} // namespace
void DefineBvhIntersect(EmitterState& s) {
if (!s.requirements.bvh) return;
if (!s.program.info.bvh) return;
const auto u = TypeU32(s), f = TypeF32(s), b = TypeBool(s), wide = TypeU64(s);
const auto vec4 = TypeU32Vector(s, 4), vec3 = TypeF32Vector(s, 3);
const auto signature = s.builder.Type(spv::OpTypeFunction, vec4, vec4, wide, f, vec3, vec3, vec3);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp
+2 -2
View File
@@ -10,7 +10,7 @@ namespace {
bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) {
const auto register_index = IR::RegIndex(reg);
const auto& registers = state.program.bindings.user_data_registers;
const auto& registers = state.program.info.user_data_registers;
const auto found = std::lower_bound(registers.begin(), registers.end(), register_index);
if (found == registers.end() || *found != register_index) {
return false;
@@ -472,7 +472,7 @@ void EmitSetAttribute(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state;
const auto& exp = ctx.Export(inst);
const auto exec = ctx.Arg(inst, 1);
if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
if (state.program.stage == ShaderType::Pixel && exp.vm && state.program.info.pixel_valid_mask &&
state.pixel_valid_mask_variable != 0) {
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1),
src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp
+4 -7
View File
@@ -867,7 +867,6 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto resource = image.indirect_resources[ordinal];
const auto& candidate = state.program.info.images[resource];
const auto kind = *IR::DescriptorBindingForImage(candidate);
const auto& binding = *IR::FindBinding(state.program.bindings, kind);
if (!runs.empty()) {
auto& run = runs.back();
const auto& first = state.program.info.images[run.resource];
@@ -875,17 +874,16 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
if (candidate.dimension == first.dimension && candidate.cube == first.cube &&
IR::DescriptorBindingForImage(first) == kind &&
candidate.mip_count == 1u && first.mip_count == 1u &&
(ordinal == 1u || (next_slot < binding.resources.size() &&
binding.resources[next_slot] == resource))) {
(ordinal == 1u || candidate.descriptor_index == next_slot)) {
// Native roots precede appended children; only the root may have a slot gap.
if (ordinal == 1u)
run.slot_bias = ResourceForDescriptor(state, kind, resource) - ordinal;
run.slot_bias = candidate.descriptor_index - ordinal;
++run.count;
continue;
}
}
runs.push_back({ordinal, 1u, resource,
ResourceForDescriptor(state, kind, resource) - ordinal});
candidate.descriptor_index - ordinal});
}
const auto EmitRun = [&](uint32_t index) {
const auto& run = runs[index];
@@ -893,8 +891,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
auto slot = Binary(state, spv::OpIAdd, TypeU32(state), selected,
ConstantU32(state, run.slot_bias));
if (run.first == 0u) {
const auto kind = *IR::DescriptorBindingForImage(image);
const auto root_slot = ResourceForDescriptor(state, kind, mem.resource);
const auto root_slot = image.descriptor_index;
if (root_slot != run.slot_bias) {
const auto is_root = Binary(state, spv::OpIEqual, TypeBool(state), selected,
ConstantU32(state, 0u));
src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h
+5 -32
View File
@@ -6,7 +6,6 @@
#include "graphics/shader/recompiler/BufferFormat.h"
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include <algorithm>
@@ -58,42 +57,21 @@ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
struct SpirvRequirements {
bool bvh = false;
bool subgroup_ballot = false;
bool subgroup_barrier = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
bool compute_derivatives = false;
bool image_gather_extended = false;
bool function_lds = false;
bool function_scratch = false;
bool pixel_valid_mask = false;
bool buffer_int64_atomics = false;
bool buffer_u8 = false;
bool buffer_u16 = false;
bool shared_int64_atomics = false;
bool coherent_buffers = false;
bool float64 = false;
};
SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
struct EmitterState {
EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
EmitterState(IR::Program& program_, ShaderStageInputInfo input_info_)
: builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
program(program_), input_info(input_info_),
requirements(AnalyzeProgramRequirements(program_)) {
program(program_), input_info(input_info_) {
if (ShaderWorkgroupInput(program.stage, input_info) != nullptr) {
lds_storage_class = spv::StorageClassWorkgroup;
}
if (IR::FindBinding(program.bindings, IR::DescriptorBindingKind::SharedMemory) != nullptr) {
if (program.info.uses_lds && program.stage == ShaderType::Compute &&
input_info.compute != nullptr && input_info.compute->lds_storage) {
lds_storage_class = spv::StorageClassStorageBuffer;
}
}
Builder builder;
const IR::Program& program;
IR::Program& program;
ShaderStageInputInfo input_info;
uint32_t void_type = 0;
uint32_t bool_type = 0;
@@ -113,7 +91,6 @@ struct EmitterState {
uint32_t tess_inner_variable = 0;
uint32_t tess_patch_base = 0;
const SpirvRequirements requirements;
uint32_t lane_count = 1;
uint32_t lane_half = 0;
uint32_t storage_buffer_variable = 0;
@@ -313,9 +290,6 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
IR::DescriptorBindingKind kind, uint32_t resource,
const char* reason);
uint32_t ResourceForDescriptor(const EmitterState& state, IR::DescriptorBindingKind kind,
uint32_t resource);
uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
@@ -410,7 +384,6 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem);
void EmitMemoryOffsets(EmitterState& state);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp
+1 -2
View File
@@ -251,8 +251,7 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
if (mem.kind == IR::ResourceKind::Buffer) {
const auto resource = ResourceForDescriptor(ctx.state, IR::DescriptorBindingKind::Buffers,
mem.resource);
const auto resource = ctx.state.program.info.buffers.at(mem.resource).descriptor_index;
const auto prefix = ctx.state.memory_byte_offsets[resource];
const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state),
BufferByteAddress(ctx, inst, mem), prefix);
src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp
+6 -14
View File
@@ -102,17 +102,9 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
return state.program.info.buffers[mem.resource].descriptor_format;
}
uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) {
if (!mem.formatted) return mem.data_bits;
const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
: program.info.buffers[mem.resource].descriptor_format;
const auto info = Format::GetFormatInfo(format);
return info.type == Format::ComponentType::Unknown || info.packed_bitfield
? 32u : info.component_bits[0];
}
void EmitMemoryOffsets(EmitterState& state) {
for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
const auto count = state.program.bindings.descriptor_counts[static_cast<size_t>(IR::DescriptorBindingKind::Buffers)];
for (uint32_t i = 0; i < count; i++) {
const auto word =
EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
const auto shift = ConstantU32(state, (i % 4u) * 8u);
@@ -146,7 +138,7 @@ void EnsureLdsStorage(EmitterState& state) {
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
return variable;
};
if (state.requirements.shared_int64_atomics) {
if (state.program.info.shared_int64_atomics) {
state.lds_variable = define(TypeU32(state), 4u);
state.lds_u64_variable = define(TypeU64(state), 8u);
state.builder.AddName(state.lds_u64_variable, "lds_qwords");
@@ -167,7 +159,7 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
"storage buffer descriptor array was not emitted");
}
const auto array_index =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
state.program.info.buffers.at(mem.resource).descriptor_index;
MemoryResourceAccess access {
.kind = mem.kind,
.memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone};
@@ -214,7 +206,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
case IR::ResourceKind::ScalarBuffer:
case IR::ResourceKind::Buffer: {
const auto bits = mem.kind == IR::ResourceKind::Buffer
? StorageBufferElementBits(state.program, mem) : 32u;
? IR::StorageBufferElementBits(state.program, mem) : 32u;
const auto variable = bits == 8u ? state.storage_buffer_u8_variable
: bits == 16u ? state.storage_buffer_u16_variable
: state.storage_buffer_variable;
@@ -245,7 +237,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
TypeStorageBufferElementPointer(state, 32));
}
const auto pointer = state.builder.AllocateId();
if (access.kind == IR::ResourceKind::Lds && state.requirements.shared_int64_atomics) {
if (access.kind == IR::ResourceKind::Lds && state.program.info.shared_int64_atomics) {
state.builder.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, storage_class),
pointer, access.object_pointer, ConstantU32(state, 0), index);
return pointer;
src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp
+75 -35
View File
@@ -200,6 +200,43 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
}
void DefineDescriptors(EmitterState& state) {
using Kind = IR::DescriptorBindingKind;
auto& info = state.program.info;
auto& bindings = state.program.bindings;
auto& counts = bindings.descriptor_counts;
auto& buffer_count = counts[static_cast<size_t>(Kind::Buffers)];
for (uint32_t index = 0; index < info.buffers.size(); ++index) {
info.buffers[index].descriptor_index = (info.live_buffers & (uint64_t{1} << index)) != 0
? buffer_count++ : UINT32_MAX;
}
std::array<const IR::ImageResource*, IR::ImageBindingCount> image_types {};
for (auto& image: info.images) {
const auto kind = IR::DescriptorBindingForImage(image);
if (!kind.has_value() || image.mip_count == 0u ||
(image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
EXIT("shader image has an invalid descriptor type or mip count");
}
auto& count = counts[static_cast<size_t>(*kind)];
image.descriptor_index = count;
if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = &image;
count += image.mip_count;
}
counts[static_cast<size_t>(Kind::Samplers)] = static_cast<uint32_t>(info.samplers.size());
counts[static_cast<size_t>(Kind::Gds)] = info.uses_gds;
counts[static_cast<size_t>(Kind::SharedMemory)] =
info.uses_lds && state.lds_storage_class == spv::StorageClassStorageBuffer;
counts[static_cast<size_t>(Kind::BdaPagetable)] = info.uses_dma;
counts[static_cast<size_t>(Kind::FaultBuffer)] = info.uses_dma;
counts[static_cast<size_t>(Kind::FlattenedSrt)] = info.uses_flattened_srt;
bindings.memory_offset_dword = static_cast<uint32_t>(info.user_data_registers.size());
if (info.uses_dispatch_threads) {
bindings.dispatch_thread_dword = bindings.memory_offset_dword;
bindings.memory_offset_dword += 3u;
}
bindings.push_data_start_dword =
IR::PushData::StartFor(bindings.push_data_start_dword, bindings.ShaderDataDwords());
counts[static_cast<size_t>(Kind::ShaderData)] =
bindings.ShaderDataDwords() != 0 && !bindings.UsesPushData();
if (state.program.bindings.UsesPushData() || state.program.stage == ShaderType::Mesh) {
const auto type = PushConstantBlockType(state);
state.push_constant_variable = state.builder.DefineGlobalVariable(
@@ -207,7 +244,10 @@ void DefineDescriptors(EmitterState& state) {
state.builder.AddName(type, "BufferResource");
state.builder.AddName(state.push_constant_variable, "vsharp");
}
for (const auto& binding: state.program.bindings.descriptors) {
for (uint32_t index = 0; index < counts.size(); ++index) {
if (counts[index] == 0) continue;
bindings.descriptor_mask |= uint64_t{1} << index;
const auto kind = static_cast<Kind>(index);
const auto Define = [&](uint32_t type, const char* name,
spv::StorageClass storage = spv::StorageClassStorageBuffer) {
const auto variable =
@@ -215,27 +255,27 @@ void DefineDescriptors(EmitterState& state) {
state.builder.AddName(variable, name);
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0);
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding,
IR::NativeBinding(state.program.stage, binding.kind));
IR::NativeBinding(state.program.stage, kind));
return variable;
};
const auto ArrayType = [&](uint32_t type) {
return state.builder.Type(
spv::OpTypeArray, type,
ConstantU32(state, static_cast<uint32_t>(binding.resources.size())));
ConstantU32(state, counts[index]));
};
switch (binding.kind) {
switch (kind) {
case IR::DescriptorBindingKind::Buffers:
state.storage_buffer_variable =
Define(ArrayType(StorageBufferType(state)), "buffers");
if (state.requirements.buffer_u8) {
if (state.program.info.buffer_u8) {
state.storage_buffer_u8_variable =
Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8");
}
if (state.requirements.buffer_u16) {
if (state.program.info.buffer_u16) {
state.storage_buffer_u16_variable =
Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16");
}
if (state.requirements.buffer_int64_atomics) {
if (state.program.info.buffer_int64_atomics) {
state.storage_buffer_u64_variable =
Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64");
}
@@ -248,7 +288,7 @@ void DefineDescriptors(EmitterState& state) {
}
// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
// must participate in visibility for cache-bypassing polling loads.
if (state.requirements.coherent_buffers) {
if (state.program.info.coherent_buffers) {
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent);
}
}
@@ -276,7 +316,7 @@ void DefineDescriptors(EmitterState& state) {
case IR::DescriptorBindingKind::SharedMemory:
state.lds_variable = Define(StorageBufferType(state), "lds_dwords");
state.builder.AddAnnotation(spv::OpDecorate, state.lds_variable, spv::DecorationCoherent);
if (state.requirements.shared_int64_atomics) {
if (state.program.info.shared_int64_atomics) {
state.lds_u64_variable = Define(StorageBufferType(state, 64), "lds_qwords");
state.builder.AddAnnotation(spv::OpDecorate, state.lds_u64_variable, spv::DecorationCoherent);
for (const auto variable: {state.lds_variable, state.lds_u64_variable}) {
@@ -285,11 +325,11 @@ void DefineDescriptors(EmitterState& state) {
}
break;
default: {
EXIT_IF(IR::ImageBindingResourceClass(binding.kind) ==
EXIT_IF(IR::ImageBindingResourceClass(kind) ==
IR::ImageResourceClass::None);
const auto& image = state.program.info.images.at(binding.resources.front());
const auto name = "image_" + std::to_string(static_cast<uint32_t>(binding.kind));
state.image_variables[IR::ImageBindingIndex(binding.kind)] =
const auto& image = *image_types[IR::ImageBindingIndex(kind)];
const auto name = "image_" + std::to_string(static_cast<uint32_t>(kind));
state.image_variables[IR::ImageBindingIndex(kind)] =
Define(ArrayType(ImageType(state, image)), name.c_str(),
spv::StorageClassUniformConstant);
if (image.dimension == ImageDimension::Dim1D ||
@@ -437,7 +477,7 @@ void DefineInputs(EmitterState& state) {
state.inputs.push_back({{kind, 0, components, name}});
}
};
if (IR::FindBinding(state.program.bindings, IR::DescriptorBindingKind::SharedMemory) != nullptr) {
if (state.program.info.uses_lds && state.lds_storage_class == spv::StorageClassStorageBuffer) {
add_builtin(IR::StageInputKind::WorkgroupId, 3, "gl_WorkGroupID");
add_builtin(IR::StageInputKind::NumWorkgroups, 3, "gl_NumWorkGroups");
}
@@ -524,7 +564,7 @@ void DefineInputs(EmitterState& state) {
builtin);
}
}
if (state.requirements.subgroup_local_invocation_id) {
if (state.program.info.subgroup_local_invocation_id) {
const auto variable = DefineInterfaceVariable(state, TypeU32(state), spv::StorageClassInput,
"gl_SubgroupInvocationID");
state.subgroup_local_invocation_id_variable = variable;
@@ -647,10 +687,10 @@ void DefineModule(EmitterState& state) {
DefineOutputs(state);
DefineTessellationInterfaces(state);
DefineDescriptors(state);
if (state.requirements.function_lds) {
if (state.program.info.function_lds) {
state.lds_variable = state.builder.AllocateId();
}
if (state.requirements.function_scratch) {
if (state.program.info.function_scratch) {
for (uint32_t half = 0; half < state.lane_count; half++) {
state.scratch_variable[half] = state.builder.AllocateId();
}
@@ -675,11 +715,11 @@ void DefineModule(EmitterState& state) {
state.builder.RequireCapability(spv::CapabilityShader);
state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
if (state.requirements.buffer_u8) {
if (state.program.info.buffer_u8) {
state.builder.RequireExtension("SPV_KHR_8bit_storage");
state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess);
}
if (state.requirements.buffer_u16) {
if (state.program.info.buffer_u16) {
state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
}
if (state.program.info.uses_dma) {
@@ -687,11 +727,11 @@ void DefineModule(EmitterState& state) {
state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
}
if (state.requirements.buffer_int64_atomics || state.requirements.shared_int64_atomics) {
if (state.program.info.buffer_int64_atomics || state.program.info.shared_int64_atomics) {
state.builder.RequireCapability(spv::CapabilityInt64);
state.builder.RequireCapability(spv::CapabilityInt64Atomics);
}
if (state.requirements.shared_int64_atomics &&
if (state.program.info.shared_int64_atomics &&
state.lds_storage_class == spv::StorageClassWorkgroup) {
state.builder.RequireVersion(0x00010400u);
state.builder.RequireExtension("SPV_KHR_workgroup_memory_explicit_layout");
@@ -714,21 +754,21 @@ void DefineModule(EmitterState& state) {
if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) {
state.builder.RequireCapability(spv::CapabilitySampleRateShading);
}
if (state.requirements.image_gather_extended) {
if (state.program.info.image_gather_extended) {
state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
}
if (state.lane_count == 2 || state.requirements.subgroup_barrier ||
state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
state.requirements.subgroup_local_invocation_id) {
if (state.lane_count == 2 || state.program.info.subgroup_barrier ||
state.program.info.subgroup_ballot || state.program.info.subgroup_shuffle ||
state.program.info.subgroup_local_invocation_id) {
state.builder.RequireCapability(spv::CapabilityGroupNonUniform);
}
if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
if (state.lane_count == 2 || state.program.info.subgroup_ballot) {
state.builder.RequireCapability(spv::CapabilityGroupNonUniformBallot);
}
if (state.requirements.subgroup_shuffle) {
if (state.program.info.subgroup_shuffle) {
state.builder.RequireCapability(spv::CapabilityGroupNonUniformShuffle);
}
if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) {
if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) {
state.builder.RequireCapability(spv::CapabilityComputeDerivativeGroupQuadsKHR);
state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives");
}
@@ -751,7 +791,7 @@ void DefineModule(EmitterState& state) {
// contract prevents host compilers from treating synthesized IEEE values as finite.
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
32u);
if (state.requirements.float64) {
if (state.program.info.float64) {
EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
state.input_info.compute->float_mode != 0xc0);
// Use native rounding for MODE=0xc0, consistent with ordinary FP32 arithmetic.
@@ -763,8 +803,8 @@ void DefineModule(EmitterState& state) {
// state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u);
}
if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) {
uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_x = state.program.info.compute_derivatives ? 2u : 1u;
uint32_t local_y = state.program.info.compute_derivatives ? 2u : 1u;
uint32_t local_z = 1u;
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
@@ -772,7 +812,7 @@ void DefineModule(EmitterState& state) {
if (state.lane_count == 2) {
local_x = ((local_x * local_y * local_z + 63u) / 64u) * 32u;
local_y = local_z = 1u;
if (state.requirements.compute_derivatives) {
if (state.program.info.compute_derivatives) {
local_y = local_x / 2u;
local_x = 2u;
}
@@ -791,14 +831,14 @@ void DefineModule(EmitterState& state) {
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeEarlyFragmentTests);
}
}
if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) {
if (state.program.info.compute_derivatives && state.program.stage == ShaderType::Compute) {
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDerivativeGroupQuadsKHR);
}
state.builder.AddName(state.main_func, "main");
if (state.requirements.function_lds) {
if (state.program.info.function_lds) {
state.builder.AddName(state.lds_variable, "lds_dwords");
}
if (state.requirements.function_scratch) {
if (state.program.info.function_scratch) {
for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddName(state.scratch_variable[half], "scratch_dwords");
}
src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp
+7 -12
View File
@@ -618,7 +618,7 @@ void EmitProgram(EmitterState& state) {
high.half = 1;
}
std::optional<DispatcherFunctionState> dispatcher;
if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
if (state.program.stage == ShaderType::Pixel && state.program.info.pixel_valid_mask) {
state.pixel_valid_mask_variable = state.builder.AllocateId();
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
}
@@ -686,28 +686,23 @@ void EmitProgram(EmitterState& state) {
}
DefineGetBdaPointer(state);
DefineBvhIntersect(state);
for (const auto* block: program.blocks) {
if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32;
})) {
ctx.scratch_u32_variable = state.builder.AllocateId();
if (state.lane_count == 2) {
high.scratch_u32_variable = state.builder.AllocateId();
}
break;
if (program.info.uses_swizzle) {
ctx.scratch_u32_variable = state.builder.AllocateId();
if (state.lane_count == 2) {
high.scratch_u32_variable = state.builder.AllocateId();
}
}
state.builder.AddFunction(spv::OpFunction, TypeVoid(state),
state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
spv::FunctionControlMaskNone, TypeFunction(state));
EmitLabel(state, state.entry_label);
if (state.requirements.function_lds) {
if (state.program.info.function_lds) {
state.builder.AddFunction(
spv::OpVariable,
TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)),
state.lds_variable, spv::StorageClassFunction);
}
if (state.requirements.function_scratch) {
if (state.program.info.function_scratch) {
for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddFunction(
spv::OpVariable,