#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h" #include "common/assert.h" #include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h" #include "graphics/shader/recompiler/ir/ShaderIR.h" #include #include namespace Libs::Graphics::ShaderRecompiler::Spirv { namespace { [[noreturn]] void Fail(const IR::Program& program, const char* reason) { EXIT("SPIR-V validation failed: hash=0x%016" PRIx64 " stage=%u reason=%s\n", program.shader_hash, static_cast(program.stage), reason); std::abort(); } void ValidateNativeProgram(const IR::Program& program) { using Kind = IR::DescriptorBindingKind; constexpr auto KindCount = static_cast(Kind::Count); std::array, KindCount> expected; std::array present {}; const auto Dense = [](size_t size) { std::vector values(size); for (uint32_t i = 0; i < values.size(); i++) { values[i] = i; } return values; }; auto Expect = [&](Kind kind, std::vector resources = {}) { const auto index = static_cast(kind); present[index] = true; expected[index] = std::move(resources); }; if (!program.info.buffers.empty()) { Expect(Kind::Buffers, Dense(program.info.buffers.size())); } for (uint32_t i = 0; i < program.info.images.size(); i++) { const auto kind = IR::DescriptorBindingForImage(program.info.images[i]); if (!kind.has_value()) { Fail(program, "native shader plan has an invalid image class"); } present[static_cast(*kind)] = true; const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage; const auto count = dynamic ? program.info.images[i].mip_count : 1u; if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) { Fail(program, "native shader plan has an invalid image mip descriptor count"); } expected[static_cast(*kind)].insert(expected[static_cast(*kind)].end(), count, i); } if (!program.info.samplers.empty()) { Expect(Kind::Samplers, Dense(program.info.samplers.size())); } bool uses_gds = false; for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) { continue; } const auto index = inst.Flags().index; if (index >= program.memory_info.size()) { Fail(program, "shared operation has invalid memory metadata"); } const auto kind = program.memory_info[index].kind; if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { Fail(program, "shared operation has invalid resource kind"); } uses_gds |= kind == IR::ResourceKind::Gds; } } if (uses_gds) { Expect(Kind::Gds); } if (program.info.uses_dma) { Expect(Kind::BdaPagetable); Expect(Kind::FaultBuffer); } const bool uses_flattened_runtime = !program.srt_reads.empty() || std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) { return image.indirect_search_iterations != 0u; }); if (uses_flattened_runtime) { Expect(Kind::FlattenedSrt); } if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) { Expect(Kind::ShaderData); } std::array seen {}; for (const auto& binding: program.bindings.descriptors) { const auto kind = static_cast(binding.kind); if (kind >= KindCount || seen[kind] || !present[kind] || binding.resources != expected[kind]) { Fail(program, "native descriptor groups do not match shader topology"); } seen[kind] = true; } for (size_t i = 0; i < KindCount; i++) { if (present[i] != seen[i]) { Fail(program, "native shader plan is missing a required descriptor group"); } } const auto has_shader_data_storage = present[static_cast(Kind::ShaderData)]; const auto shader_data_dwords = program.bindings.ShaderDataDwords(); if ((program.bindings.UsesPushData() && !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) || program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() || program.bindings.memory_offset_count != program.info.buffers.size() || has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) || !std::is_sorted(program.bindings.user_data_registers.begin(), program.bindings.user_data_registers.end()) || std::adjacent_find(program.bindings.user_data_registers.begin(), program.bindings.user_data_registers.end()) != program.bindings.user_data_registers.end()) { Fail(program, "native shader-data layout is inconsistent"); } const auto planning_only_handle = [&](const IR::Inst& handle) { return !handle.Uses().empty() && std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) { const auto op = use.user->GetOpcode(); if (op != IR::ValueOpcode::LoadAddressU32 && op != IR::ValueOpcode::ReadConstBuffer) { return false; } const auto index = use.user->Flags().index; return index < program.memory_info.size() && program.memory_info[index].planning_only; }); }; const auto indirect_buffer_handle = [&](const IR::Inst& handle) { return program.info.uses_dma && handle.NumArgs() == 4u && !handle.Uses().empty() && std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) { if (IR::BufferAccessOf(use.user->GetOpcode()) != IR::BufferAccess::Read) { return false; } const auto index = use.user->Flags().index; return index < program.memory_info.size() && program.memory_info[index].kind == IR::ResourceKind::IndirectBuffer; }); }; for (const auto* block: program.blocks) { for (const auto& inst: *block) { const auto dense = inst.Flags(); switch (inst.GetOpcode()) { case IR::ValueOpcode::GetBufferResource: if (planning_only_handle(inst) || indirect_buffer_handle(inst)) { break; } if (dense >= program.info.buffers.size()) { Fail(program, "typed buffer handle has an invalid dense resource"); } break; case IR::ValueOpcode::GetAddressResource: if (planning_only_handle(inst)) { break; } if (inst.NumArgs() != 2 || !program.info.uses_dma) { Fail(program, "typed address handle has invalid DMA metadata"); } break; case IR::ValueOpcode::GetScratchResource: if (inst.NumArgs() != 0 || program.scratch_dwords == 0) { Fail(program, "typed scratch handle has invalid shader metadata"); } break; case IR::ValueOpcode::GetImageResource: if (dense >= program.info.images.size()) { Fail(program, "typed image handle has an invalid dense resource"); } break; case IR::ValueOpcode::GetSamplerResource: if (dense >= program.info.samplers.size()) { Fail(program, "typed sampler handle has an invalid dense resource"); } break; case IR::ValueOpcode::ReadConst: { const auto slot = inst.Arg(1).Resolve(); if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 || slot.U32() >= program.srt_reads.size()) { Fail(program, "flattened SRT read has an invalid dense slot"); } break; } default: break; } } } } } // namespace Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) { SpirvRequirements requirements {}; for (const auto* block: program.blocks) { for (const auto& inst: *block) { if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic && inst.GetType() == IR::Type::U64) { requirements.buffer_int64_atomics = true; } const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access; if (address_access != IR::AddressAccess::None) { const auto memory_index = inst.Flags().index; if (memory_index >= program.memory_info.size()) { Fail(program, "address operation has invalid memory metadata"); } if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) { if (program.scratch_dwords == 0) { Fail(program, "scratch operation has no per-thread storage"); } requirements.function_scratch = true; } else if (address_access == IR::AddressAccess::Write) { Fail(program, "writable FLAT/GLOBAL addresses require GPU ownership tracking"); } } if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) { const auto memory_index = inst.Flags().index; if (memory_index >= program.memory_info.size()) { Fail(program, "buffer operation has invalid memory metadata"); } const auto& memory = program.memory_info[memory_index]; if (memory.kind == IR::ResourceKind::IndirectBuffer) { requirements.subgroup_local_invocation_id = true; } if (memory.kind == IR::ResourceKind::Buffer) { if (memory.resource >= program.info.buffers.size()) { Fail(program, "buffer operation has invalid resource metadata"); } if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) { if (program.stage != ShaderType::Compute) { Fail(program, "buffer ADD_TID is only valid for compute shaders"); } requirements.subgroup_local_invocation_id = true; } } } const auto shared_access = IR::SharedAccessOf(inst.GetOpcode()); if (shared_access != IR::SharedAccess::None) { const auto index = inst.Flags().index; if (index >= program.memory_info.size()) { Fail(program, "shared operation has invalid memory metadata"); } const auto kind = program.memory_info[index].kind; if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) { Fail(program, "shared operation has invalid resource kind"); } if (program.stage != ShaderType::Compute && program.stage != ShaderType::Mesh && kind == IR::ResourceKind::Lds) { requirements.function_lds = true; } if (shared_access == IR::SharedAccess::Append || shared_access == IR::SharedAccess::Consume) { requirements.subgroup_ballot = true; requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; } } switch (inst.GetOpcode()) { case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break; case IR::ValueOpcode::DppMoveU32: case IR::ValueOpcode::ReadFirstLane: case IR::ValueOpcode::ReadLane: { requirements.subgroup_ballot = true; requirements.subgroup_shuffle = true; if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) { requirements.subgroup_local_invocation_id = true; } break; } case IR::ValueOpcode::DppUpdateU32: case IR::ValueOpcode::WriteLane: { requirements.subgroup_ballot = true; requirements.subgroup_local_invocation_id = true; break; } case IR::ValueOpcode::Permlane16U32: { requirements.subgroup_ballot = true; requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; break; } case IR::ValueOpcode::SwizzleU32: case IR::ValueOpcode::BpermuteU32: { requirements.subgroup_ballot = true; requirements.subgroup_shuffle = true; requirements.subgroup_local_invocation_id = true; break; } case IR::ValueOpcode::LaneId: requirements.subgroup_local_invocation_id |= program.stage != ShaderType::TessellationControl; break; case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break; case IR::ValueOpcode::ImageGatherRaw: requirements.image_gather_extended = true; break; case IR::ValueOpcode::SetAttribute: { const auto index = inst.Flags().index; if (index >= program.export_info.size()) { Fail(program, "attribute export has invalid metadata"); } if (program.stage == ShaderType::Pixel && program.export_info[index].vm) { requirements.pixel_valid_mask = true; } break; } default: break; } } } return requirements; } std::vector EmitProgram(const IR::Program& program, ShaderStageInputInfo input_info) { using namespace Emitter; if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex && program.stage != ShaderType::Pixel && program.stage != ShaderType::Mesh && program.stage != ShaderType::Local && program.stage != ShaderType::TessellationControl && program.stage != ShaderType::TessellationEvaluation) { Fail(program, "binary SPIR-V emitter received an unsupported shader stage"); } if (!program.srt_plan_complete || !program.resource_tracking_complete || !program.shader_info_complete || !program.binding_layout_complete) { Fail(program, "SPIR-V emitter requires a fully planned native shader program"); } ValidateNativeProgram(program); IR::ValidateProgram(program, true); EmitterState state(program, input_info); const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info); state.lane_count = workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u ? 2u : 1u; DefineModule(state); EmitProgram(state); state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func, "main", state.interface_variables); return state.builder.Build(); } } // namespace Libs::Graphics::ShaderRecompiler::Spirv