diff --git a/CMakeLists.txt b/CMakeLists.txt index 855202ff9..f044bc533 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -431,16 +431,15 @@ add_executable(shader_vertex_metadata_tests EXCLUDE_FROM_ALL ) target_include_directories(shader_vertex_metadata_tests PRIVATE ${inc_headers}) -add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL - "${KYTY_TESTS_DIR}/ShaderStageRuntimeTests.cpp" +add_executable(resource_materialization_tests EXCLUDE_FROM_ALL + "${KYTY_TESTS_DIR}/ResourceMaterializationTests.cpp" "${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp" "${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp" - "${KYTY_SOURCE_DIR}/graphics/shader/shaderStageRuntime.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp" "${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp" ) -target_link_libraries(shader_stage_runtime_tests fmt::fmt) -target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers}) +target_link_libraries(resource_materialization_tests fmt::fmt) +target_include_directories(resource_materialization_tests PRIVATE ${inc_headers}) add_executable(resource_tracking_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp" @@ -542,7 +541,7 @@ if(BUILD_TESTING) add_test(NAME page_manager COMMAND $) add_test(NAME bit_array COMMAND $) add_test(NAME shader_vertex_metadata COMMAND $) - add_test(NAME shader_stage_runtime COMMAND $) + add_test(NAME resource_materialization COMMAND $) add_test(NAME resource_tracking COMMAND $) add_test(NAME event_queue_lifetime COMMAND $) add_test(NAME kernel_file_system COMMAND $) @@ -598,7 +597,7 @@ if(BUILD_TESTING) page_manager_tests bit_array_tests shader_vertex_metadata_tests - shader_stage_runtime_tests + resource_materialization_tests resource_tracking_tests event_queue_lifetime_tests kernel_file_system_tests diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp index 5ae9c72f9..ba81551e0 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.cpp @@ -65,7 +65,7 @@ vk::DescriptorType NativeDescriptorType(BindingKind kind) { case BindingKind::BdaPagetable: case BindingKind::FaultBuffer: case BindingKind::FlattenedSrt: - case BindingKind::UserData: return vk::DescriptorType::eStorageBuffer; + case BindingKind::ShaderData: return vk::DescriptorType::eStorageBuffer; case BindingKind::Count: EXIT("invalid native descriptor binding kind"); } EXIT("invalid native descriptor binding kind"); @@ -108,12 +108,6 @@ static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) { dst.range = 16; } -static void CopyNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& source, - std::span destination) { - EXIT_IF(source.dword_count != destination.size()); - std::copy_n(source.dwords.begin(), destination.size(), destination.begin()); -} - static Prospero::ImageType TextureType(const ShaderTextureResource& descriptor) { const auto type = descriptor.Type(); return type == Prospero::ImageType::kCube ? Prospero::ImageType::kColor2DArray : type; @@ -658,8 +652,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource, const ShaderRecompiler::IR::DescriptorValue& value) { - ShaderTextureResource descriptor; - CopyNativeDescriptor(value, descriptor.fields); + auto descriptor = DecodeNativeDescriptor(value); const bool storage = resource.written; if (storage) { ValidateStorageImageResource(resource); @@ -802,17 +795,11 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR } static vk::Sampler NativeSampler(RenderContext& context, - const ShaderRecompiler::IR::Program& program, uint32_t index, + const ShaderRecompiler::IR::CompiledShaderInfo& program, + uint32_t index, const ShaderRecompiler::IR::DescriptorValue& value) { - ShaderSamplerResource descriptor; - CopyNativeDescriptor(value, descriptor.fields); - const bool depth_compare = std::any_of(program.info.sampled_pairs.begin(), - program.info.sampled_pairs.end(), [&](const auto& pair) { - return pair.sampler == index && - pair.image < program.info.images.size() && - program.info.images[pair.image].depth_compare; - }); - if (!depth_compare) { + auto descriptor = DecodeNativeDescriptor(value); + if (!program.info.samplers[index].depth_compare) { descriptor.fields[0] &= ~(0x7u << 12u); } if (program.info.samplers[index].force_point_filtering) { @@ -869,15 +856,10 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti KYTY_PROFILER_FUNCTION(); EXIT_IF(!runtime); const auto& program = *runtime.program; - const auto& snapshot = *runtime.resources; - if (!ShaderRecompiler::IR::ValidateResourceSpecialization(program, snapshot)) { - EXIT("invalid native shader runtime snapshot: hash=0x%016" PRIx64 " stage=%u\n", - program.shader_hash, static_cast(program.stage)); - } - + const auto& snapshot = runtime.resources; PreparedBindings prepared; - prepared.program = runtime.program.get(); - prepared.snapshot = runtime.resources.get(); + prepared.program = runtime.program; + prepared.snapshot = &runtime.resources; auto& descriptors = prepared.resources; descriptors.buffers.reserve(program.info.buffers.size()); descriptors.images.reserve(program.info.images.size()); @@ -890,17 +872,11 @@ PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runti for (uint32_t i = 0; i < program.info.samplers.size(); i++) { descriptors.samplers.push_back(NativeSampler(m_context, program, i, snapshot.samplers[i])); } - if (ShaderRecompiler::IR::FindBinding( - program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) != - nullptr) { - prepared.flattened_srt.assign(snapshot.flattened_srt.begin(), snapshot.flattened_srt.end()); - } - - prepared.user_data.reserve(program.bindings.ShaderDataDwords()); + prepared.shader_data.reserve(program.bindings.ShaderDataDwords()); for (const auto reg: program.bindings.user_data_registers) { - prepared.user_data.push_back(snapshot.user_data[reg - program.user_data_base]); + prepared.shader_data.push_back(snapshot.user_data[reg - program.user_data_base]); } - prepared.user_data.resize(program.bindings.ShaderDataDwords()); + prepared.shader_data.resize(program.bindings.ShaderDataDwords()); if (ShaderRecompiler::IR::FindBinding( program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) { descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle(); @@ -915,22 +891,21 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) { const auto& snapshot = *prepared.snapshot; auto& cache = m_context.GetBufferCache(); - prepared.buffer_ids.clear(); - prepared.buffer_ids.reserve(program.info.buffers.size()); + prepared.buffer_sources.clear(); + prepared.buffer_sources.reserve(program.info.buffers.size()); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - ShaderBufferResource descriptor; - CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); + auto descriptor = DecodeNativeDescriptor(snapshot.buffers[i]); const auto address = descriptor.Base48(); const auto stride = descriptor.Stride(); const auto records = descriptor.NumRecords(); EXIT_IF(stride != 0 && records > UINT64_MAX / stride); const auto requested_size = stride != 0 ? static_cast(stride) * records : records; if (address == 0 || requested_size == 0) { - prepared.buffer_ids.emplace_back(); + prepared.buffer_sources.emplace_back(descriptor, BufferId {}); continue; } const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size); - prepared.buffer_ids.push_back(cache.FindBuffer(address, size)); + prepared.buffer_sources.emplace_back(descriptor, cache.FindBuffer(address, size)); } } @@ -942,32 +917,33 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) { const auto& snapshot = *prepared.snapshot; auto& resources = prepared.resources; const auto& layout = program.bindings; - EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size()); + EXIT_IF(prepared.buffer_sources.size() != program.info.buffers.size()); resources.buffers.clear(); resources.buffers.reserve(program.info.buffers.size()); - EXIT_IF(prepared.user_data.size() != layout.ShaderDataDwords()); - std::fill(prepared.user_data.begin() + layout.memory_offset_dword, prepared.user_data.end(), 0); + EXIT_IF(prepared.shader_data.size() != layout.ShaderDataDwords()); + std::fill(prepared.shader_data.begin() + layout.memory_offset_dword, + prepared.shader_data.end(), 0); auto pack_memory_offset = [&](uint32_t index, uint32_t offset) { const auto dword = layout.memory_offset_dword + index / 4u; const auto shift = (index % 4u) * 8u; - prepared.user_data[dword] |= offset << shift; + prepared.shader_data[dword] |= offset << shift; }; for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - ShaderBufferResource descriptor; - CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields); + const auto& [descriptor, buffer_id] = prepared.buffer_sources[i]; uint32_t buffer_offset = 0; resources.buffers.push_back(NativeStorageBuffer(m_context, descriptor, program.info.buffers[i], program.stage, i, - buffer_offset, prepared.buffer_ids[i])); + buffer_offset, buffer_id)); pack_memory_offset(i, buffer_offset); } - if (!prepared.flattened_srt.empty()) { - resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt); + if (ShaderRecompiler::IR::FindBinding( + layout, ShaderRecompiler::IR::DescriptorBindingKind::FlattenedSrt) != nullptr) { + resources.flattened_srt = NativeUpload(m_context, snapshot.flattened_srt); } if (ShaderRecompiler::IR::FindBinding( - program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) { - resources.user_data = NativeUpload(m_context, prepared.user_data); + program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::ShaderData) != nullptr) { + resources.shader_data = NativeUpload(m_context, prepared.shader_data); } } @@ -1056,14 +1032,13 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, auto vk_buffer = buffer.Handle(); size_t descriptor_count = 0; size_t write_count = 0; + ShaderRecompiler::IR::PushData push_data; + bool has_push_data = false; constexpr auto GraphicsStages = vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment; - const auto push_constant_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics - ? vk::ShaderStageFlags {GraphicsStages} - : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}; for (const auto* prepared: prepared_bindings) { EXIT_IF(prepared == nullptr || prepared->program == nullptr || - prepared->snapshot == nullptr || prepared->committed); + prepared->snapshot == nullptr); write_count += prepared->program->bindings.descriptors.size(); for (const auto& binding: prepared->program->bindings.descriptors) { descriptor_count += NativeDescriptorCount(binding); @@ -1077,7 +1052,6 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, m_descriptor_buffers.clear(); m_descriptor_images.clear(); m_descriptor_writes.clear(); - m_push_constants.fill(0); m_descriptor_buffers.reserve(descriptor_count); m_descriptor_images.reserve(descriptor_count); m_descriptor_writes.reserve(write_count); @@ -1167,13 +1141,13 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, break; } case BindingKind::FlattenedSrt: - case BindingKind::UserData: + case BindingKind::ShaderData: case BindingKind::Gds: { const BufferView* view = &descriptors.gds; if (binding.kind == BindingKind::FlattenedSrt) { view = &descriptors.flattened_srt; - } else if (binding.kind == BindingKind::UserData) { - view = &descriptors.user_data; + } else if (binding.kind == BindingKind::ShaderData) { + view = &descriptors.shader_data; } EXIT_IF(view->buffer == nullptr); m_descriptor_buffers.emplace_back(view->buffer, view->offset, view->range); @@ -1206,20 +1180,22 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, EXIT_IF(m_image_occurrences[i] != expected); } - if (program.bindings.push_constant_size != 0) { - const auto offset = program.bindings.push_constant_offset; - EXIT_IF(offset % sizeof(uint32_t) != 0 || - program.bindings.push_constant_size != - prepared->user_data.size() * sizeof(uint32_t) || - offset + program.bindings.push_constant_size > - ShaderRecompiler::IR::NativePushConstantSize); - std::copy(prepared->user_data.begin(), prepared->user_data.end(), - m_push_constants.begin() + offset / sizeof(uint32_t)); + const auto shader_data_dwords = program.bindings.ShaderDataDwords(); + EXIT_IF(prepared->shader_data.size() != shader_data_dwords); + if (program.bindings.UsesPushData()) { + std::ranges::copy(prepared->shader_data, + push_data.dwords.begin() + program.bindings.push_data_start_dword); + has_push_data = true; } } - vk_buffer.pushConstants(pipeline.pipeline_layout, push_constant_stages, 0, - ShaderRecompiler::IR::NativePushConstantSize, - m_push_constants.data()); + + if (has_push_data) { + const auto stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics + ? vk::ShaderStageFlags {GraphicsStages} + : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute}; + vk_buffer.pushConstants(pipeline.pipeline_layout, stages, 0, sizeof(push_data), + push_data.dwords.data()); + } if (!m_descriptor_writes.empty()) { EXIT_IF(pipeline.descriptor_set_layout == nullptr); @@ -1239,9 +1215,6 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer, 0, nullptr); } } - for (auto* prepared: prepared_bindings) { - prepared->committed = true; - } } } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/descriptors.h b/src/graphics/host_gpu/renderer/pipeline/descriptors.h index 3c8849a89..f53acf99d 100644 --- a/src/graphics/host_gpu/renderer/pipeline/descriptors.h +++ b/src/graphics/host_gpu/renderer/pipeline/descriptors.h @@ -40,17 +40,15 @@ struct NativeDescriptors { std::vector samplers; BufferView gds; BufferView flattened_srt; - BufferView user_data; + BufferView shader_data; }; struct PreparedBindings { - const ShaderRecompiler::IR::Program* program = nullptr; + const ShaderRecompiler::IR::CompiledShaderInfo* program = nullptr; const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr; NativeDescriptors resources; - std::vector buffer_ids; - std::vector flattened_srt; - std::vector user_data; - bool committed = false; + std::vector> buffer_sources; + std::vector shader_data; }; [[nodiscard]] vk::DescriptorType diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp index 90947b97c..e78a8376e 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp @@ -1,6 +1,7 @@ #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "common/assert.h" +#include "common/emulatorConfig.h" #include "common/file.h" #include "common/logging/log.h" #include "common/profiler.h" @@ -11,7 +12,9 @@ #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" +#include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/shaderCompiler.h" +#include "kernel/memory.h" #include "kytyGitVersion.h" #include "loader/systemContent.h" @@ -20,13 +23,14 @@ #include #include #include +#include #include #include +#include #include +#include #include #include - -#include #include namespace Libs::Graphics { @@ -84,42 +88,188 @@ void NormalizeStaticParamsForDynamicState(PipelineStaticParameters& static_param static_params.scissor_ltrb[3] = 1; } +bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) { + return value != nullptr && + Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, value, sizeof(*value)); +} + +void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash, + const std::vector& spirv) { + if (!Config::GraphicsDebugDumpEnabled()) { + return; + } + static std::atomic_int id = 0; + const auto path = Config::GetShaderLogFolder() / fmt::format("{:04d}_new_shader_{}_{:016x}.spv", + id++, stage_name, shader_hash); + Common::File::CreateDirectories(path.parent_path()); + Common::File file(path, Common::File::Mode::Write); + if (file.IsInvalid()) { + const auto path_text = Common::PathToString(path); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); + return; + } + file.Write(spirv.data(), spirv.size() * sizeof(uint32_t)); +} + +void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash, + std::span code, const std::string& decoded_dump) { + if (!Config::GraphicsDebugDumpEnabled()) { + return; + } + EXIT_IF(code.empty()); + static std::atomic_int id = 0; + const auto base = Config::GetShaderLogFolder() / "original" / + fmt::format("{:04d}_new_shader_{}_{:016x}", id++, stage_name, shader_hash); + Common::File::CreateDirectories(base.parent_path()); + for (const auto& [suffix, data, size]: { + std::tuple {".bin", static_cast(code.data()), code.size_bytes()}, + std::tuple {".rdna2", static_cast(decoded_dump.data()), + decoded_dump.size()}, + }) { + if (size == 0) { + continue; + } + auto path = base; + path += suffix; + Common::File file(path, Common::File::Mode::Write); + if (file.IsInvalid()) { + const auto path_text = Common::PathToString(path); + LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); + } else { + file.Write(data, size); + } + } +} + +bool ValidateShaderSpirv(const char* label, uint64_t shader_hash, + const std::vector& spirv) { + if (!Config::ShaderValidationEnabled()) { + return true; + } + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_3); + std::string messages; + tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, + const spv_position_t& position, const char* message) { + messages += fmt::format("{}: {} ({}) {}\n", static_cast(position.line), + static_cast(position.column), static_cast(position.index), + message); + }); + if (tools.Validate(spirv)) { + return true; + } + spvtools::SpirvTools disassembler(SPV_ENV_VULKAN_1_2); + std::string text; + disassembler.Disassemble(spirv, &text, + static_cast(SPV_BINARY_TO_TEXT_OPTION_NO_HEADER) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_COMMENT) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_INDENT) | + static_cast(SPV_BINARY_TO_TEXT_OPTION_COLOR)); + LOGF_COLOR(Log::Color::BrightRed, "%s SPIR-V validation failed hash=0x%016" PRIx64 ":\n%s", + label, shader_hash, messages.c_str()); + LOGF("%s\n", text.c_str()); + return false; +} + } // namespace struct PipelineCache::ProgramCache { - struct SourceKey { - ShaderType stage = ShaderType::Unknown; - uint64_t hash = 0; + struct ProgramKey { + ShaderType stage = ShaderType::Unknown; + uint64_t hash = 0; + uint32_t user_data_count = 0; + uint32_t code_size = 0; + std::vector static_state; - bool operator==(const SourceKey&) const = default; + bool operator==(const ProgramKey&) const = default; }; struct Permutation { - std::vector static_key; - std::shared_ptr program; - ShaderProgram handle; + ShaderRecompiler::IR::ResourceSpecialization specialization; + ShaderRecompiler::IR::CompiledShaderInfo program; + ShaderProgram handle; }; - struct SourceKeyHash { - std::size_t operator()(const SourceKey& key) const { + struct SourceEntry { + explicit SourceEntry(ShaderRecompiler::IR::ResourcePlan plan) + : resource_plan(std::move(plan)) { + permutations.reserve(8); + } + + ShaderRecompiler::IR::ResourcePlan resource_plan; + std::vector permutations; + }; + + struct ProgramKeyHash { + std::size_t operator()(const ProgramKey& key) const { std::size_t hash = static_cast(key.stage); PipelineKeyHash::Mix(hash, static_cast(key.hash)); if constexpr (sizeof(std::size_t) < sizeof(uint64_t)) { PipelineKeyHash::Mix(hash, static_cast(key.hash >> 32u)); } + PipelineKeyHash::Mix(hash, key.user_data_count); + PipelineKeyHash::Mix(hash, key.code_size); + PipelineKeyHash::Mix(hash, key.static_state.size()); + // Bucket same-shape static variants by source. ProgramKey equality performs the one + // exact state comparison needed on a stable hit without hashing up to 429 words first. return hash; } }; - static uint64_t MixId(uint64_t hash, uint64_t value) { - return hash ^ (value + 0x9e3779b97f4a7c15ull + (hash << 6u) + (hash >> 2u)); + static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13; + + template + Permutation CompilePermutation(const ShaderParams& params, + const ShaderRecompiler::CompileOptions& options, + ShaderRecompiler::TranslateResult translated, + ShaderRecompiler::IR::ResourceSpecialization specialization, + uint32_t push_data_start_dword) { + constexpr const char* stage_name = [] { + if constexpr (Stage == ShaderType::Vertex) { + return "vs"; + } else if constexpr (Stage == ShaderType::Pixel) { + return "ps"; + } else { + static_assert(Stage == ShaderType::Compute); + return "cs"; + } + }(); + auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, + specialization, push_data_start_dword); + DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump); + if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) { + DumpShaderSpirv(stage_name, options.shader_hash, result.spirv); + EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label, + options.shader_hash); + } + DumpShaderSpirv(stage_name, options.shader_hash, result.spirv); + + vk::ShaderModuleCreateInfo create_info {}; + create_info.sType = vk::StructureType::eShaderModuleCreateInfo; + create_info.codeSize = result.spirv.size() * sizeof(uint32_t); + create_info.pCode = result.spirv.data(); + vk::ShaderModule module = nullptr; + RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module), + "create recompiled shader module"); + EXIT_IF(module == nullptr); + if (options.dump_ir) { + if (!options.early_dump) { + LOGF("%s decoded RDNA2:\n%s", options.dump_label, result.decoded_dump.c_str()); + LOGF("%s IR:\n%s", options.dump_label, result.ir_dump.c_str()); + } + LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", options.dump_label, + static_cast(result.spirv.size()), options.wave_size); + } + return { + .specialization = std::move(specialization), + .program = std::move(result.program).TakeCompiledInfo(), + .handle = {.id = ++next_shader_id, .module = module}, + }; } - static constexpr std::size_t MaxStaticKeyWords = - 12 + ShaderVertexInputInfo::RES_MAX * 17; - template - ShaderProgram Get(const ShaderParams& params, InputInfo& input_info) { + ShaderProgram Get(const ShaderParams& params, InputInfo& input_info, + uint32_t& push_data_cursor) { constexpr ShaderType stage = [] { if constexpr (std::is_same_v) { return ShaderType::Vertex; @@ -131,36 +281,106 @@ struct PipelineCache::ProgramCache { } }(); - BuildStageStaticKey(input_info, key_scratch); - auto& permutations = programs[{stage, params.hash}]; - for (const auto& permutation: permutations) { - if (permutation.static_key == key_scratch && - MaterializeProgram(permutation.program, params, input_info)) { - return permutation.handle; + lookup_key.stage = stage; + lookup_key.hash = params.hash; + lookup_key.user_data_count = static_cast(params.user_data.size()); + lookup_key.code_size = static_cast(params.code.size()); + BuildStageStaticKey(input_info, lookup_key.static_state); + auto entry = programs.find(lookup_key); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime { + .user_data = params.user_data, + .shader_base = params.Base(), + .read_specialization_memory = ReadShaderGuestMemory, + }; + if (entry != programs.end()) { + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( + entry->second.resource_plan, runtime, resources, specialization)); + if (const auto permutation = std::ranges::find_if( + entry->second.permutations, [&](const Permutation& candidate) { + const auto& layout = candidate.program.bindings; + return layout.push_data_start_dword == + ShaderRecompiler::IR::PushData::StartFor( + push_data_cursor, layout.ShaderDataDwords()) && + candidate.specialization == specialization; + }); + permutation != entry->second.permutations.end()) { + input_info.stage = {.program = &permutation->program, + .resources = std::move(resources)}; + permutation->program.bindings.AdvancePushData(push_data_cursor); + return permutation->handle; } } - const auto module = CompileProgram(device, params, input_info); - EXIT_IF(module == nullptr || !input_info.stage); - uint64_t id = MixId(MixId(params.hash, static_cast(stage)), permutations.size()); - if (id == 0) { - id = 1; + ShaderStageInputInfo stage_input {}; + if constexpr (std::is_same_v) { + stage_input.vertex = &input_info; + } else if constexpr (std::is_same_v) { + stage_input.pixel = &input_info; + } else { + stage_input.compute = &input_info; } - const ShaderProgram handle {.id = id, .module = module}; - permutations.push_back({key_scratch, input_info.stage.program, handle}); + constexpr const char* label = [] { + if constexpr (stage == ShaderType::Vertex) { + return "ShaderRecompiler VS"; + } else if constexpr (stage == ShaderType::Pixel) { + return "ShaderRecompiler PS"; + } else { + return "ShaderRecompiler CS"; + } + }(); + ShaderRecompiler::CompileOptions options; + options.stage = stage; + options.shader_hash = params.hash; + options.user_data = params.user_data; + options.dump_ir = Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent; + options.early_dump = options.dump_ir; + options.dump_label = label; + options.input_info = stage_input; + if constexpr (stage == ShaderType::Vertex) { + options.user_data_base = 8; + options.scratch_dwords = input_info.scratch_size_dwords; + } else if constexpr (stage == ShaderType::Pixel) { + options.scratch_dwords = input_info.scratch_size_dwords; + } else { + options.scratch_dwords = input_info.scratch_size_dwords; + options.wave_size = input_info.wave_size; + } + auto translated = ShaderRecompiler::TranslateProgram(params.code, options); + if (entry == programs.end()) { + auto resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(resource_plan, runtime, resources, + specialization)); + entry = programs.try_emplace(lookup_key, std::move(resource_plan)).first; + } + entry->second.permutations.push_back(CompilePermutation( + params, options, std::move(translated), std::move(specialization), push_data_cursor)); + const auto& permutation = entry->second.permutations.back(); + input_info.stage = {.program = &permutation.program, .resources = std::move(resources)}; + permutation.program.bindings.AdvancePushData(push_data_cursor); std::printf("Num compiled %u shaders\n", ++num_compiled); - return handle; + return permutation.handle; } explicit ProgramCache(vk::Device device): device(device) { - key_scratch.reserve(MaxStaticKeyWords); + lookup_key.static_state.reserve(MaxStaticKeyWords); + } + ~ProgramCache() { + for (const auto& [key, entry]: programs) { + (void)key; + for (const auto& permutation: entry.permutations) { + device.destroyShaderModule(permutation.handle.module, nullptr); + } + } } - std::unordered_map, SourceKeyHash> programs; - std::vector key_scratch; - vk::Device device; - uint32_t num_compiled = 0; + std::unordered_map programs; + ProgramKey lookup_key; + vk::Device device; + uint32_t num_compiled = 0; + uint64_t next_shader_id = 0; }; PipelineCache::PipelineCache(GraphicContext& graphics) @@ -181,12 +401,6 @@ PipelineCache::~PipelineCache() { }; destroy(m_graphics_pipelines); destroy(m_compute_pipelines); - for (const auto& [key, permutations]: m_program_cache->programs) { - (void)key; - for (const auto& permutation: permutations) { - m_graphics.device.destroyShaderModule(permutation.handle.module, nullptr); - } - } if (m_driver_cache != nullptr) { m_graphics.device.destroyPipelineCache(m_driver_cache, nullptr); } @@ -212,7 +426,7 @@ void PipelineCache::InitializeDriverCache() { return; } - m_driver_cache_path = std::filesystem::path("_PipelineCache") / (title_id + ".bin"); + m_driver_cache_path = std::filesystem::path("_PipelineCache") / (title_id + ".bin"); const auto path = Common::PathToString(m_driver_cache_path); const bool cache_exists = Common::File::IsFileExisting(m_driver_cache_path); if (cache_exists) { @@ -224,7 +438,7 @@ void PipelineCache::InitializeDriverCache() { if (cache_exists) { Common::File file(m_driver_cache_path, Common::File::Mode::Read); const auto file_size = file.IsInvalid() ? 0 : file.Size(); - const auto signature = DriverCacheSignature(m_graphics.GetPhysicalDeviceProperties()); + const auto signature = DriverCacheSignature(m_graphics.GetPhysicalDeviceProperties()); if (file_size >= signature.size() + sizeof(uint64_t) && file_size <= std::numeric_limits::max()) { std::string cached_signature(signature.size(), '\0'); @@ -236,7 +450,8 @@ void PipelineCache::InitializeDriverCache() { file.Read(cached_signature.data(), static_cast(cached_signature.size()), &signature_read); file.Read(&payload_hash, sizeof(payload_hash), &hash_read); - file.Read(initial_data.data(), static_cast(initial_data.size()), &payload_read); + file.Read(initial_data.data(), static_cast(initial_data.size()), + &payload_read); file.Close(); if (signature_read != cached_signature.size() || hash_read != sizeof(payload_hash) || payload_read != initial_data.size() || cached_signature != signature || @@ -248,8 +463,7 @@ void PipelineCache::InitializeDriverCache() { } } else { file.Close(); - PipelineCacheLog("Vulkan pipeline cache: invalidating {} (invalid file size)", - path); + PipelineCacheLog("Vulkan pipeline cache: invalidating {} (invalid file size)", path); } } @@ -308,7 +522,7 @@ void PipelineCache::Save() { return; } payload.resize(size); - auto prefix = DriverCacheSignature(m_graphics.GetPhysicalDeviceProperties()); + auto prefix = DriverCacheSignature(m_graphics.GetPhysicalDeviceProperties()); const auto payload_hash = XXH3_64bits(payload.data(), payload.size()); prefix.append(reinterpret_cast(&payload_hash), sizeof(payload_hash)); if (!Common::File::CreateDirectories(m_driver_cache_path.parent_path())) { @@ -338,15 +552,20 @@ void PipelineCache::Save() { m_driver_cache = nullptr; } -ShaderProgram PipelineCache::GetVertexProgram(const HW::VertexShaderInfo& regs, - const HW::ShaderRegisters& sh, - const HW::Context& context, - ShaderVertexInputInfo& input_info) { - const auto params = PrepareProgram(regs, sh, input_info); +PipelineCache::GraphicsPrograms PipelineCache::GetGraphicsPrograms( + const HW::VertexShaderInfo& vertex_regs, const HW::PixelShaderInfo& pixel_regs, + const HW::ShaderRegisters& sh, const HW::Context& context, + std::span target_export_mapping, + bool pixel_active, ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info) { + const auto vertex_params = PrepareProgram(vertex_regs, sh, vertex_info); + ShaderParams pixel_params; + if (pixel_active) { + pixel_params = PrepareProgram(pixel_regs, sh, target_export_mapping, pixel_info); + } if (context.GetClipControl().clip_disable) { const auto& viewport = context.GetScreenViewport().viewports[0]; const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; - auto& clip = input_info.clip_space; + auto& clip = vertex_info.clip_space; clip.scale[0] = viewport.xscale; clip.scale[1] = viewport.yscale; clip.offset[0] = viewport.xoffset; @@ -358,26 +577,23 @@ ShaderProgram PipelineCache::GetVertexProgram(const HW::VertexShaderInfo& regs, clip.enabled = true; } Common::LockGuard lock(m_mutex); - return m_program_cache->Get(params, input_info); -} - -ShaderProgram PipelineCache::GetPixelProgram( - const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vertex_info, - std::span target_export_mapping, - ShaderPixelInputInfo& input_info) { - const auto params = PrepareProgram(regs, sh, vertex_info, target_export_mapping, input_info); - Common::LockGuard lock(m_mutex); - return m_program_cache->Get(params, input_info); + uint32_t push_data_cursor = 0; + GraphicsPrograms result; + if (pixel_active) { + result.pixel = m_program_cache->Get(pixel_params, pixel_info, push_data_cursor); + } + result.vertex = m_program_cache->Get(vertex_params, vertex_info, push_data_cursor); + return result; } ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderComputeInputInfo& input_info) { input_info.needs_lds_barriers = !m_graphics.compute_wave64_supported; - const auto params = PrepareProgram(regs, sh, input_info); + const auto params = PrepareProgram(regs, sh, input_info); Common::LockGuard lock(m_mutex); - return m_program_cache->Get(params, input_info); + uint32_t push_data_cursor = 0; + return m_program_cache->Get(params, input_info, push_data_cursor); } bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept { @@ -401,8 +617,7 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( Common::LockGuard lock(m_mutex); auto& ctx = command.GetRegisters(); - const HW::BlendColor& bclr = ctx.GetBlendColor(); - uint32_t color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {}; + uint32_t color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {}; for (uint32_t i = 0; i < color_count; i++) { color_mask[i] = (colors[i].image_id ? colors[i].export_mapping.ApplyMask(render_target_mask_slot( @@ -506,11 +721,6 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( static_params.blend_enable[i] = bc.enable; static_params.blend_bypass[i] = rt.info.blend_bypass; } - static_params.blend_color_red = bclr.red; - static_params.blend_color_green = bclr.green; - static_params.blend_color_blue = bclr.blue; - static_params.blend_color_alpha = bclr.alpha; - NormalizeStaticParamsForDynamicState(static_params); GraphicsPipelineKey key {}; @@ -518,6 +728,30 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( key.vs_shader_id = p.vs_shader_id; key.ps_shader_id = p.ps_shader_id; key.static_params = static_params; + EXIT_IF(vs_input_info.buffers_num < 0 || + vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX || + vs_input_info.resources_num < 0 || + vs_input_info.resources_num > ShaderVertexInputInfo::RES_MAX); + key.vertex_input.binding_count = static_cast(vs_input_info.buffers_num); + key.vertex_input.attribute_count = static_cast(vs_input_info.resources_num); + uint32_t attributes_num = 0; + for (int binding = 0; binding < vs_input_info.buffers_num; binding++) { + const auto& buffer = vs_input_info.buffers[binding]; + EXIT_IF(buffer.attr_num < 0 || buffer.attr_num > ShaderVertexInputBuffer::ATTR_MAX); + attributes_num += static_cast(buffer.attr_num); + EXIT_IF(attributes_num > static_cast(vs_input_info.resources_num)); + key.vertex_input.bindings[binding] = {.stride = buffer.stride, + .instance = buffer.fetch_index != 0}; + for (int attribute = 0; attribute < buffer.attr_num; attribute++) { + const auto index = buffer.attr_indices[attribute]; + EXIT_IF(index < 0 || index >= vs_input_info.resources_num); + key.vertex_input.attributes[index] = { + .offset = buffer.attr_offsets[attribute], + .binding = static_cast(binding), + }; + } + } + EXIT_IF(attributes_num != static_cast(vs_input_info.resources_num)); if (auto iter = m_graphics_pipelines.find(key); iter != m_graphics_pipelines.end()) { return *iter->second; @@ -528,16 +762,16 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline( if (ps_active) { ShaderDbgDumpInputInfo(*ps_input_info); } - LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64 - " module=%p\n", + LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64 " module=%p\n", vs_id, static_cast(vertex_program.module), ps_id, static_cast(pixel_program.module)); } auto cached = std::make_unique(p); LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id); - CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vertex_program.module, - ps_input_info, pixel_program.module, static_params, m_driver_cache); + CreatePipelineInternal(m_graphics, *cached, rendering, key.vertex_input, vs_input_info, + vertex_program.module, ps_input_info, pixel_program.module, + static_params, m_driver_cache); LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id); EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr); diff --git a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h index 911fb33a9..12fc49ca1 100644 --- a/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h +++ b/src/graphics/host_gpu/renderer/pipeline/pipelineCache.h @@ -65,10 +65,6 @@ struct PipelineStaticParameters { bool separate_alpha_blend[RENDER_COLOR_ATTACHMENTS_MAX] = {}; bool blend_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {}; bool blend_bypass[RENDER_COLOR_ATTACHMENTS_MAX] = {}; - float blend_color_red = 0.0f; - float blend_color_green = 0.0f; - float blend_color_blue = 0.0f; - float blend_color_alpha = 0.0f; bool operator==(const PipelineStaticParameters& other) const noexcept; }; @@ -85,7 +81,7 @@ static_assert(sizeof(PipelineStaticParameters) == sizeof(bool) + sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) + sizeof(uint32_t[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(bool) * 3 + sizeof(uint8_t[RENDER_COLOR_ATTACHMENTS_MAX]) * 6 + - sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) * 3 + sizeof(float) * 4); + sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) * 3); struct PipelineRenderingState { std::array color_formats {}; @@ -96,13 +92,31 @@ struct PipelineRenderingState { bool operator==(const PipelineRenderingState&) const = default; }; +struct PipelineVertexInputState { + struct Binding { + uint32_t stride = 0; + bool instance = false; + bool operator==(const Binding&) const = default; + }; + struct Attribute { + uint32_t offset = 0; + uint8_t binding = 0; + bool operator==(const Attribute&) const = default; + }; + + std::array bindings {}; + std::array attributes {}; + uint8_t binding_count = 0; + uint8_t attribute_count = 0; + + bool operator==(const PipelineVertexInputState&) const = default; +}; + struct ShaderProgram { uint64_t id = 0; vk::ShaderModule module = nullptr; - explicit operator bool() const { - return id != 0 && module != nullptr; - } + explicit operator bool() const { return id != 0 && module != nullptr; } }; class PipelineCache { @@ -128,25 +142,26 @@ public: uint64_t cs_shader_id = 0; }; - ShaderProgram GetVertexProgram(const HW::VertexShaderInfo& regs, - const HW::ShaderRegisters& sh, - const HW::Context& context, - ShaderVertexInputInfo& input_info); - ShaderProgram - GetPixelProgram(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vertex_info, - std::span target_export_mapping, - ShaderPixelInputInfo& input_info); - ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs, - const HW::ShaderRegisters& sh, - ShaderComputeInputInfo& input_info); + struct GraphicsPrograms { + ShaderProgram vertex; + ShaderProgram pixel; + }; - GraphicsPipeline& CreateGraphicsPipeline( - std::span colors, const RenderDepthInfo& depth, - const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command, - const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, - bool primitive_restart_enable, const ShaderProgram& vertex_program, - const ShaderProgram& pixel_program); + GraphicsPrograms GetGraphicsPrograms( + const HW::VertexShaderInfo& vertex_regs, const HW::PixelShaderInfo& pixel_regs, + const HW::ShaderRegisters& sh, const HW::Context& context, + std::span target_export_mapping, + bool pixel_active, ShaderVertexInputInfo& vertex_info, ShaderPixelInputInfo& pixel_info); + ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs, + const HW::ShaderRegisters& sh, + ShaderComputeInputInfo& input_info); + + GraphicsPipeline& + CreateGraphicsPipeline(std::span colors, const RenderDepthInfo& depth, + const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command, + const ShaderPixelInputInfo* ps_input_info, + vk::PrimitiveTopology topology, bool primitive_restart_enable, + const ShaderProgram& vertex_program, const ShaderProgram& pixel_program); ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info, const ShaderProgram& compute_program); @@ -157,11 +172,13 @@ private: PipelineRenderingState rendering; uint64_t vs_shader_id = 0; uint64_t ps_shader_id = 0; + PipelineVertexInputState vertex_input; PipelineStaticParameters static_params; bool operator==(const GraphicsPipelineKey& other) const { return rendering == other.rendering && vs_shader_id == other.vs_shader_id && - ps_shader_id == other.ps_shader_id && static_params == other.static_params; + ps_shader_id == other.ps_shader_id && vertex_input == other.vertex_input && + static_params == other.static_params; } }; @@ -202,6 +219,16 @@ private: PipelineKeyHash::MixRendering(hash, key.rendering); PipelineKeyHash::Mix(hash, key.vs_shader_id); PipelineKeyHash::Mix(hash, key.ps_shader_id); + PipelineKeyHash::Mix(hash, key.vertex_input.binding_count); + for (uint32_t i = 0; i < key.vertex_input.binding_count; i++) { + PipelineKeyHash::Mix(hash, key.vertex_input.bindings[i].stride); + PipelineKeyHash::Mix(hash, key.vertex_input.bindings[i].instance); + } + PipelineKeyHash::Mix(hash, key.vertex_input.attribute_count); + for (uint32_t i = 0; i < key.vertex_input.attribute_count; i++) { + PipelineKeyHash::Mix(hash, key.vertex_input.attributes[i].offset); + PipelineKeyHash::Mix(hash, key.vertex_input.attributes[i].binding); + } PipelineKeyHash::MixStaticParams(hash, key.static_params); return hash; } @@ -215,7 +242,7 @@ private: } }; - GraphicContext& m_graphics; + GraphicContext& m_graphics; std::unique_ptr m_program_cache; vk::PipelineCache m_driver_cache = nullptr; std::filesystem::path m_driver_cache_path; @@ -232,13 +259,13 @@ private: void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id); void CreatePipelineInternal( GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline, - const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, - vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info, - vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params, - vk::PipelineCache driver_cache); + const PipelineRenderingState& rendering, const PipelineVertexInputState& vertex_input, + const ShaderVertexInputInfo& vs_input_info, vk::ShaderModule vertex_module, + const ShaderPixelInputInfo* ps_input_info, vk::ShaderModule pixel_module, + const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache); void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline, const ShaderComputeInputInfo& input_info, - vk::ShaderModule compute_module, vk::PipelineCache driver_cache); + vk::ShaderModule compute_module, vk::PipelineCache driver_cache); } // namespace Libs::Graphics diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp index fdd0ea815..5c3a237b0 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.cpp @@ -69,7 +69,7 @@ vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer) { } std::vector -CollectShaderBufferWrites(const ShaderRecompiler::IR::Program& program, +CollectShaderBufferWrites(const ShaderRecompiler::IR::CompiledShaderInfo& program, const ShaderRecompiler::IR::ResourceSnapshot& resources) { EXIT_IF(resources.buffers.size() != program.info.buffers.size()); std::vector writes; @@ -97,7 +97,7 @@ CollectShaderBufferWrites(const ShaderRecompiler::IR::Program& program, bool HasShaderBufferWrites(const ShaderStageRuntime& runtime) { EXIT_IF(!runtime); - return !CollectShaderBufferWrites(*runtime.program, *runtime.resources).empty(); + return !CollectShaderBufferWrites(*runtime.program, runtime.resources).empty(); } void ShaderAccessBarrier(vk::CommandBuffer vk_buffer, vk::PipelineStageFlags source_stages) { diff --git a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h index 27643fe20..70a1c7286 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h +++ b/src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h @@ -23,7 +23,7 @@ VulkanMemoryBarrier MakeShaderWriteHazardDependency(); VulkanMemoryBarrier MakeShaderWriteDependency(); vk::BufferMemoryBarrier MakeGdsDependency(vk::Buffer buffer); std::vector -CollectShaderBufferWrites(const ShaderRecompiler::IR::Program& program, +CollectShaderBufferWrites(const ShaderRecompiler::IR::CompiledShaderInfo& program, const ShaderRecompiler::IR::ResourceSnapshot& resources); bool HasShaderBufferWrites(const ShaderStageRuntime& runtime); void ShaderAccessBarrier(vk::CommandBuffer vk_buffer, vk::PipelineStageFlags source_stages); diff --git a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp index 06b8ac6c5..833d830ef 100644 --- a/src/graphics/host_gpu/renderer/pipeline/shaders.cpp +++ b/src/graphics/host_gpu/renderer/pipeline/shaders.cpp @@ -390,7 +390,7 @@ static vk::BlendOp GetBlendOp(uint32_t op) { } static void AddLayoutBindings(std::vector& descriptor_bindings, - const ShaderRecompiler::IR::Program& program, + const ShaderRecompiler::IR::CompiledShaderInfo& program, vk::ShaderStageFlagBits stage) { for (const auto& binding: program.bindings.descriptors) { descriptor_bindings.push_back( @@ -421,10 +421,10 @@ static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipe // NOLINTNEXTLINE(readability-function-cognitive-complexity) void CreatePipelineInternal( GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline, - const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info, - vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info, - vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params, - vk::PipelineCache driver_cache) { + const PipelineRenderingState& rendering, const PipelineVertexInputState& vertex_input, + const ShaderVertexInputInfo& vs_input_info, vk::ShaderModule vertex_module, + const ShaderPixelInputInfo* ps_input_info, vk::ShaderModule pixel_module, + const PipelineStaticParameters& static_params, vk::PipelineCache driver_cache) { const bool ps_active = ps_input_info != nullptr; EXIT_IF(vertex_module == nullptr || (ps_active && pixel_module == nullptr)); @@ -505,135 +505,131 @@ void CreatePipelineInternal( shader_stages[shader_stage_count++] = frag_shader_stage_info; } - vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX]; - vk::VertexInputBindingDescription input_desc[ShaderVertexInputInfo::RES_MAX]; + vk::VertexInputAttributeDescription input_attr[ShaderVertexInputInfo::RES_MAX] {}; + vk::VertexInputBindingDescription input_desc[ShaderVertexInputInfo::RES_MAX] {}; - for (int bi = 0; bi < vs_input_info.buffers_num; bi++) { - const auto& b = vs_input_info.buffers[bi]; - input_desc[bi].binding = bi; - input_desc[bi].stride = b.stride; - input_desc[bi].inputRate = - (b.fetch_index == 0 ? vk::VertexInputRate::eVertex : vk::VertexInputRate::eInstance); + for (uint32_t binding = 0; binding < vertex_input.binding_count; binding++) { + input_desc[binding].binding = binding; + input_desc[binding].stride = vertex_input.bindings[binding].stride; + input_desc[binding].inputRate = vertex_input.bindings[binding].instance + ? vk::VertexInputRate::eInstance + : vk::VertexInputRate::eVertex; + } + for (uint32_t index = 0; index < vertex_input.attribute_count; index++) { + input_attr[index].binding = vertex_input.attributes[index].binding; + input_attr[index].location = index; + input_attr[index].offset = vertex_input.attributes[index].offset; - for (int ai = 0; ai < b.attr_num; ai++) { - auto index = b.attr_indices[ai]; - input_attr[index].binding = bi; - input_attr[index].location = index; - input_attr[index].offset = b.attr_offsets[ai]; + uint32_t attr_size = 4; + const auto registers_num = vs_input_info.resources_dst[index].registers_num; + const auto compiled_components = + vs_input_info.stage.program->info.vertex_fetch_components[index]; + const auto used_components = + compiled_components > 0 ? static_cast(compiled_components) : registers_num; + GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size, + static_cast(used_components)); - uint32_t attr_size = 4; - auto registers_num = vs_input_info.resources_dst[index].registers_num; - const auto compiled_components = - vs_input_info.stage.program->info.vertex_fetch_components[index]; - auto used_components = - compiled_components > 0 ? static_cast(compiled_components) : registers_num; - GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size, - static_cast(used_components)); - - if (graphics_debug_dump_enabled()) { - static std::atomic_uint log_count = 0; - const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); - if (log_id < 128) { - LOGF("VertexInputState[%u]: attr=%u binding=%u offset=%u stride=%u fmt=%d " - "src_fmt=%u dst=v%u regs=%u" - " fetched_components=%u attr_size=%u swizzle=%u,%u,%u,%u\n", - log_id, index, input_attr[index].binding, input_attr[index].offset, - input_desc[bi].stride, static_cast(input_attr[index].format), - static_cast(vs_input_info.resources[index].Format()), - static_cast(vs_input_info.resources_dst[index].register_start), - static_cast(registers_num), - static_cast(used_components), attr_size, - static_cast(vs_input_info.resources[index].DstSelX()), - static_cast(vs_input_info.resources[index].DstSelY()), - static_cast(vs_input_info.resources[index].DstSelZ()), - static_cast(vs_input_info.resources[index].DstSelW())); - } + if (graphics_debug_dump_enabled()) { + static std::atomic_uint log_count = 0; + const auto log_id = log_count.fetch_add(1, std::memory_order_relaxed); + if (log_id < 128) { + LOGF("VertexInputState[%u]: attr=%u binding=%u offset=%u stride=%u fmt=%d " + "src_fmt=%u dst=v%u regs=%u" + " fetched_components=%u attr_size=%u swizzle=%u,%u,%u,%u\n", + log_id, index, input_attr[index].binding, input_attr[index].offset, + input_desc[input_attr[index].binding].stride, + static_cast(input_attr[index].format), + static_cast(vs_input_info.resources[index].Format()), + static_cast(vs_input_info.resources_dst[index].register_start), + static_cast(registers_num), static_cast(used_components), + attr_size, static_cast(vs_input_info.resources[index].DstSelX()), + static_cast(vs_input_info.resources[index].DstSelY()), + static_cast(vs_input_info.resources[index].DstSelZ()), + static_cast(vs_input_info.resources[index].DstSelW())); } + } - if (vs_input_info.resources[index].OutOfBounds() != 0) { - static bool logged = false; - if (!logged) { - LOGF("VertexInput: temporary: accepting PS5 out-of-bounds behavior %" PRIu8 - "\n", - vs_input_info.resources[index].OutOfBounds()); - logged = true; - } + if (vs_input_info.resources[index].OutOfBounds() != 0) { + static bool logged = false; + if (!logged) { + LOGF("VertexInput: temporary: accepting PS5 out-of-bounds behavior %" PRIu8 "\n", + vs_input_info.resources[index].OutOfBounds()); + logged = true; } + } - EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].AddTid()); - EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].SwizzleEnabled()); + EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].AddTid()); + EXIT_NOT_IMPLEMENTED(vs_input_info.resources[index].SwizzleEnabled()); - auto log_unsupported_vertex_swizzle = [index, attr_size, registers_num]( - uint32_t swizzle, uint32_t expected) { - static bool logged = false; - if (!logged) { - LOGF( - "VertexInput: temporary: accepting unsupported dst swizzle at attr %" PRIu32 - " (attr_size=%" PRIu32 ", regs=%" PRIu32 ", swizzle=0x%03" PRIx32 - ", expected=0x%03" PRIx32 ")\n", - static_cast(index), attr_size, registers_num, swizzle, expected); - logged = true; - } - }; - - switch (registers_num) { - case 1: { - auto swizzle = vs_input_info.resources[index].DstSelX(); - if (swizzle != DstSel(4)) { - log_unsupported_vertex_swizzle(swizzle, DstSel(4)); - } - break; - } - case 2: { - auto swizzle = vs_input_info.resources[index].DstSelXY(); - auto expected = (attr_size == 1 ? DstSel(4, 0) : DstSel(4, 5)); - if (swizzle != expected) { - log_unsupported_vertex_swizzle(swizzle, expected); - } - break; - } - case 3: { - auto swizzle = vs_input_info.resources[index].DstSelXYZ(); - auto expected = DstSel(4, 5, 6); - switch (attr_size) { - case 1: expected = DstSel(4, 0, 0); break; - case 2: expected = DstSel(4, 5, 0); break; - default: break; - } - if (swizzle != expected) { - log_unsupported_vertex_swizzle(swizzle, expected); - } - break; - } - case 4: { - auto swizzle = vs_input_info.resources[index].DstSelXYZW(); - auto expected = DstSel(4, 5, 6, 7); - bool supported = false; - switch (attr_size) { - case 1: - expected = DstSel(4, 0, 0, 1); - supported = (swizzle == expected); - break; - case 2: - expected = DstSel(4, 5, 0, 1); - supported = (swizzle == expected); - break; - case 3: - expected = DstSel(4, 5, 6, 1); - supported = (swizzle == expected || swizzle == DstSel(4, 5, 6, 0)); - break; - default: - supported = (swizzle == expected || swizzle == DstSel(4, 5, 6, 1) || - swizzle == DstSel(4, 5, 6, 0)); - break; - } - if (!supported) { - log_unsupported_vertex_swizzle(swizzle, expected); - } - break; - } - default: EXIT("invalid registers_num"); + auto log_unsupported_vertex_swizzle = [index, attr_size, registers_num](uint32_t swizzle, + uint32_t expected) { + static bool logged = false; + if (!logged) { + LOGF("VertexInput: temporary: accepting unsupported dst swizzle at attr %" PRIu32 + " (attr_size=%" PRIu32 ", regs=%" PRIu32 ", swizzle=0x%03" PRIx32 + ", expected=0x%03" PRIx32 ")\n", + static_cast(index), attr_size, registers_num, swizzle, expected); + logged = true; } + }; + + switch (registers_num) { + case 1: { + auto swizzle = vs_input_info.resources[index].DstSelX(); + if (swizzle != DstSel(4)) { + log_unsupported_vertex_swizzle(swizzle, DstSel(4)); + } + break; + } + case 2: { + auto swizzle = vs_input_info.resources[index].DstSelXY(); + auto expected = (attr_size == 1 ? DstSel(4, 0) : DstSel(4, 5)); + if (swizzle != expected) { + log_unsupported_vertex_swizzle(swizzle, expected); + } + break; + } + case 3: { + auto swizzle = vs_input_info.resources[index].DstSelXYZ(); + auto expected = DstSel(4, 5, 6); + switch (attr_size) { + case 1: expected = DstSel(4, 0, 0); break; + case 2: expected = DstSel(4, 5, 0); break; + default: break; + } + if (swizzle != expected) { + log_unsupported_vertex_swizzle(swizzle, expected); + } + break; + } + case 4: { + auto swizzle = vs_input_info.resources[index].DstSelXYZW(); + auto expected = DstSel(4, 5, 6, 7); + bool supported = false; + switch (attr_size) { + case 1: + expected = DstSel(4, 0, 0, 1); + supported = (swizzle == expected); + break; + case 2: + expected = DstSel(4, 5, 0, 1); + supported = (swizzle == expected); + break; + case 3: + expected = DstSel(4, 5, 6, 1); + supported = (swizzle == expected || swizzle == DstSel(4, 5, 6, 0)); + break; + default: + supported = (swizzle == expected || swizzle == DstSel(4, 5, 6, 1) || + swizzle == DstSel(4, 5, 6, 0)); + break; + } + if (!supported) { + log_unsupported_vertex_swizzle(swizzle, expected); + } + break; + } + default: EXIT("invalid registers_num"); } } @@ -641,9 +637,9 @@ void CreatePipelineInternal( vertex_input_info.sType = vk::StructureType::ePipelineVertexInputStateCreateInfo; vertex_input_info.pNext = nullptr; vertex_input_info.flags = {}; - vertex_input_info.vertexBindingDescriptionCount = vs_input_info.buffers_num; + vertex_input_info.vertexBindingDescriptionCount = vertex_input.binding_count; vertex_input_info.pVertexBindingDescriptions = input_desc; - vertex_input_info.vertexAttributeDescriptionCount = vs_input_info.resources_num; + vertex_input_info.vertexAttributeDescriptionCount = vertex_input.attribute_count; vertex_input_info.pVertexAttributeDescriptions = input_attr; vk::PipelineInputAssemblyStateCreateInfo input_assembly {}; @@ -792,15 +788,11 @@ void CreatePipelineInternal( #else color_blending.pNext = &color_write; #endif - color_blending.flags = {}; - color_blending.logicOpEnable = VK_FALSE; - color_blending.logicOp = vk::LogicOp::eCopy; - color_blending.attachmentCount = static_params.color_count; - color_blending.pAttachments = color_blend_attachment; - color_blending.blendConstants[0] = static_params.blend_color_red; - color_blending.blendConstants[1] = static_params.blend_color_green; - color_blending.blendConstants[2] = static_params.blend_color_blue; - color_blending.blendConstants[3] = static_params.blend_color_alpha; + color_blending.flags = {}; + color_blending.logicOpEnable = VK_FALSE; + color_blending.logicOp = vk::LogicOp::eCopy; + color_blending.attachmentCount = static_params.color_count; + color_blending.pAttachments = color_blend_attachment; EXIT_IF(!vs_input_info.stage); std::vector descriptor_bindings; @@ -814,8 +806,8 @@ void CreatePipelineInternal( CreateDescriptorLayout(graphics, pipeline, descriptor_bindings); constexpr auto GraphicsStages = vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment; - const vk::PushConstantRange push_constants { - GraphicsStages, 0, ShaderRecompiler::IR::NativePushConstantSize}; + const vk::PushConstantRange push_constants {GraphicsStages, 0, + ShaderRecompiler::IR::NativePushConstantSize}; vk::PipelineLayoutCreateInfo pipeline_layout_info {}; pipeline_layout_info.sType = vk::StructureType::ePipelineLayoutCreateInfo; @@ -877,6 +869,7 @@ void CreatePipelineInternal( vk::DynamicState::eStencilCompareMask, vk::DynamicState::eStencilReference, vk::DynamicState::eStencilWriteMask, + vk::DynamicState::eBlendConstants, #if !defined(__APPLE__) // Keep last so depth-only pipelines can omit this dynamic state. vk::DynamicState::eColorWriteEnableEXT, // unsupported by MoltenVK; static mask instead diff --git a/src/graphics/host_gpu/renderer/render.h b/src/graphics/host_gpu/renderer/render.h index a1c1764c8..af0972b25 100644 --- a/src/graphics/host_gpu/renderer/render.h +++ b/src/graphics/host_gpu/renderer/render.h @@ -190,8 +190,6 @@ private: std::vector m_descriptor_images; std::vector m_descriptor_writes; std::vector m_image_occurrences; - std::array - m_push_constants {}; friend struct RenderExecutorTestAccess; }; diff --git a/src/graphics/host_gpu/renderer/renderCompute.cpp b/src/graphics/host_gpu/renderer/renderCompute.cpp index 5ac238d02..b0238875b 100644 --- a/src/graphics/host_gpu/renderer/renderCompute.cpp +++ b/src/graphics/host_gpu/renderer/renderCompute.cpp @@ -46,7 +46,7 @@ static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) { bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input, const CommandBuffer& buffer) { const auto& program = *input.stage.program; - const auto& resources = *input.stage.resources; + const auto& resources = input.stage.resources; if (resources.buffers.size() != program.info.buffers.size()) { EXIT("compute runtime buffer count does not match shader metadata\n"); } @@ -81,7 +81,7 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou ShaderBufferResource& resolved_descriptor, uint32_t& resolved_clear, uint64_t& resolved_size) { const auto& program = *input.stage.program; - const auto& resources = *input.stage.resources; + const auto& resources = input.stage.resources; if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 || !program.info.images.empty() || !program.info.samplers.empty() || program.info.uses_dma || !resources.images.empty() || !resources.samplers.empty()) { @@ -224,7 +224,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, const bool large_workgroup = (input_info.threads_num[0] * input_info.threads_num[1] * input_info.threads_num[2] >= 512); const auto& program = *input_info.stage.program; - const auto& resources = *input_info.stage.resources; + const auto& resources = input_info.stage.resources; if (TryConsumeComputeMetaClear(input_info, buffer)) { ResetBindings(); return; @@ -249,7 +249,10 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, thread_group_z, mode, input_info.threads_num[0], input_info.threads_num[1], input_info.threads_num[2], program.info.buffers.size(), program.info.images.size(), sampled_images, program.info.images.size() - sampled_images, - program.info.samplers.size(), program.bindings.push_constant_size); + program.info.samplers.size(), + program.bindings.UsesPushData() + ? static_cast(sizeof(ShaderRecompiler::IR::PushData)) + : 0u); for (uint32_t i = 0; i < program.info.buffers.size(); i++) { const auto& buffer = program.info.buffers[i]; const auto r = DecodeNativeDescriptor(resources.buffers[i]); diff --git a/src/graphics/host_gpu/renderer/renderDraw.cpp b/src/graphics/host_gpu/renderer/renderDraw.cpp index acd6e6f2f..0c63cc278 100644 --- a/src/graphics/host_gpu/renderer/renderDraw.cpp +++ b/src/graphics/host_gpu/renderer/renderDraw.cpp @@ -52,7 +52,7 @@ int32_t ResolveVertexOffset(uint32_t index_offset, const ShaderVertexInputInfo& EXIT_IF(!vs_input_info.stage); const auto& program = *vs_input_info.stage.program; - const auto& resources = *vs_input_info.stage.resources; + const auto& resources = vs_input_info.stage.resources; if (program.info.vertex_offset_sgpr >= static_cast(program.user_data_base)) { const auto index = static_cast(program.info.vertex_offset_sgpr) - program.user_data_base; @@ -364,6 +364,9 @@ static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuf line_width = 1.0f; } vk_buffer.setLineWidth(line_width); + const auto& blend = ctx.GetBlendColor(); + const std::array blend_constants {blend.red, blend.green, blend.blue, blend.alpha}; + vk_buffer.setBlendConstants(blend_constants.data()); const auto& mode = ctx.GetModeControl(); const auto& poly_offset = ctx.GetPolyOffset(); @@ -495,8 +498,7 @@ struct DrawRenderState { RenderState rendering; ShaderVertexInputInfo vs_input_info; ShaderPixelInputInfo ps_input_info; - ShaderProgram vertex_program; - ShaderProgram pixel_program; + PipelineCache::GraphicsPrograms programs; }; struct DrawCallInfo { @@ -1008,30 +1010,20 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool const auto& pixel_shader_info = sh_ctx.GetPs(); const auto& shader_regs = ctx.GetShaderRegisters(); - state.vertex_program = {}; - state.pixel_program = {}; - state.ps_input_info = {}; + state.programs = {}; + state.ps_input_info = {}; std::array target_export_mapping {}; for (uint32_t i = 0; i < state.color_count; i++) { target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping; } - if (log_phases) { - LogDrawPhase(draw.name, "GetVertexProgram"); - } auto& pipeline_cache = buffer.GetContext().GetPipelineCache(); - state.vertex_program = pipeline_cache.GetVertexProgram(vertex_shader_info, shader_regs, ctx, - state.vs_input_info); - - if (!state.ps_active) { - return; - } if (log_phases) { - LogDrawPhase(draw.name, "GetPixelProgram"); + LogDrawPhase(draw.name, "GetGraphicsPrograms"); } - state.pixel_program = - pipeline_cache.GetPixelProgram(pixel_shader_info, shader_regs, state.vs_input_info, - target_export_mapping, state.ps_input_info); + state.programs = pipeline_cache.GetGraphicsPrograms( + vertex_shader_info, pixel_shader_info, shader_regs, ctx, target_export_mapping, + state.ps_active, state.vs_input_info, state.ps_input_info); } static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, CommandBuffer& buffer, @@ -1185,7 +1177,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buff auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline( std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer, state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable, - state.vertex_program, state.pixel_program); + state.programs.vertex, state.programs.pixel); // Resource preparation above may synchronously finish and restart the scheduler. From this // point onward, every operation targets the current command buffer and cannot touch guest @@ -1452,7 +1444,8 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, uint32_ const bool rect_list = topology == vk::PrimitiveTopology::ePatchList; if (rect_list && state.vs_input_info.buffers_num == 0 && - state.vs_input_info.param_export_mask == 0 && state.ps_input_info.input_num != 0) { + state.vs_input_info.stage.program->param_export_mask == 0 && + state.ps_input_info.input_num != 0) { if (graphics_debug_dump_enabled()) { LOGF("DrawIndexAuto: skipping rect-list draw with no VS param exports and PS inputs: " "ps_inputs=%u ps=0x%016" PRIx64 " es=0x%016" PRIx64 " gs=0x%016" PRIx64 "\n", diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.cpp b/src/graphics/shader/recompiler/ShaderRecompiler.cpp index 5fee8e66d..c15f89416 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.cpp +++ b/src/graphics/shader/recompiler/ShaderRecompiler.cpp @@ -34,9 +34,9 @@ const char* GetDumpLabel(const CompileOptions& options) { return options.dump_label != nullptr ? options.dump_label : "ShaderRecompiler"; } -std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) { +std::string MakeIrDump(std::string_view cfg, const IR::Program& ir) { std::string dump = "CFG:\n"; - dump += CFG::GraphToString(cfg); + dump += cfg; dump += "\nIR:\n"; dump += fmt::format("mode={} scratch_dwords={}\n", ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords); @@ -464,7 +464,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded } // namespace -CompileResult Recompile(std::span code, const CompileOptions& options) { +TranslateResult TranslateProgram(std::span code, const CompileOptions& options) { if (code.empty()) { EXIT("shader recompiler input is empty\n"); } @@ -618,30 +618,42 @@ CompileResult Recompile(std::span code, const CompileOptions& op ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr; } - const IR::SrtRuntime runtime { - .user_data = options.user_data, - .shader_base = reinterpret_cast(code.data()), - .read_memory = options.read_memory, - .userdata = options.read_memory_data, - .read_specialization_memory = options.read_specialization_memory, - }; - IR::ResourceSnapshot resources; - if (!IR::MaterializeResources(ir, runtime, resources)) { - EXIT("shader resource materialization failed: stage=%s hash=0x%016" PRIx64 "\n", - StageName(options.stage), options.shader_hash); + TranslateResult result; + result.program = std::move(ir); + if (options.dump_ir) { + result.decoded_dump = std::move(decoded_dump); + result.cfg_dump = CFG::GraphToString(cfg); + } + return result; +} + +CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options, + const IR::ResourceSpecialization& specialization, + uint32_t push_data_start_dword) { + const auto emit_begin = std::chrono::steady_clock::now(); + auto& ir = translated.program; + IR::ApplyResourceSpecialization(ir, specialization); + + const ShaderVertexInputInfo* vertex = nullptr; + const ShaderPixelInputInfo* pixel = nullptr; + const ShaderComputeInputInfo* compute = nullptr; + switch (options.stage) { + case ShaderType::Vertex: vertex = options.input_info.vertex; break; + case ShaderType::Pixel: pixel = options.input_info.pixel; break; + case ShaderType::Compute: compute = options.input_info.compute; break; + default: EXIT("invalid shader stage\n"); } - IR::SpecializeResources(ir, resources); IR::ShaderInfoOptions info_options; info_options.vertex = vertex; info_options.pixel = pixel; info_options.compute = compute; IR::CollectShaderInfo(ir, info_options); - IR::AllocateBindings(ir, options.push_constant_offset); + IR::AllocateBindings(ir, push_data_start_dword); Spirv::AnalyzeProgramRequirements(ir); std::string ir_dump; if (options.dump_ir) { - ir_dump = MakeIrDump(cfg, ir); + ir_dump = MakeIrDump(translated.cfg_dump, ir); if (options.early_dump) { LOGF("%s native IR and bindings (early):\n%s", GetDumpLabel(options), ir_dump.c_str()); } @@ -649,17 +661,19 @@ CompileResult Recompile(std::span code, const CompileOptions& op LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash); - auto spirv = Spirv::EmitProgram(ir, resources, options.input_info); + auto spirv = Spirv::EmitProgram(ir, options.input_info); LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 " elapsed_ms=%" PRIu64 "\n", GetDumpLabel(options), StageName(options.stage), options.shader_hash, - static_cast(spirv.size()), phase_ms()); + static_cast(spirv.size()), + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - emit_begin) + .count())); CompileResult result; - result.spirv = std::move(spirv); - result.program = std::move(ir); - result.resources = std::move(resources); + result.spirv = std::move(spirv); + result.program = std::move(ir); if (options.dump_ir) { - result.decoded_dump = std::move(decoded_dump); + result.decoded_dump = std::move(translated.decoded_dump); result.ir_dump = std::move(ir_dump); } return result; diff --git a/src/graphics/shader/recompiler/ShaderRecompiler.h b/src/graphics/shader/recompiler/ShaderRecompiler.h index 8846f3e64..94ff50776 100644 --- a/src/graphics/shader/recompiler/ShaderRecompiler.h +++ b/src/graphics/shader/recompiler/ShaderRecompiler.h @@ -17,27 +17,32 @@ struct CompileOptions { uint32_t user_data_base = 0; uint32_t scratch_dwords = 0; uint64_t shader_hash = 0; - uint32_t push_constant_offset = 0; bool dump_ir = true; bool early_dump = false; const char* dump_label = nullptr; std::span user_data; - IR::SrtMemoryReader read_memory = nullptr; - IR::SrtMemoryReader read_specialization_memory = nullptr; - void* read_memory_data = nullptr; ShaderStageInputInfo input_info; }; -struct CompileResult { - std::vector spirv; - std::string decoded_dump; - std::string ir_dump; - IR::Program program; - IR::ResourceSnapshot resources; +struct TranslateResult { + IR::Program program; + std::string decoded_dump; + std::string cfg_dump; }; -[[nodiscard]] CompileResult Recompile(std::span code, - const CompileOptions& options); +struct CompileResult { + std::vector spirv; + std::string decoded_dump; + std::string ir_dump; + IR::Program program; +}; + +[[nodiscard]] TranslateResult TranslateProgram(std::span code, + const CompileOptions& options); +[[nodiscard]] CompileResult CompileProgram(TranslateResult translated, + const CompileOptions& options, + const IR::ResourceSpecialization& specialization, + uint32_t push_data_start_dword = 0); } // namespace Libs::Graphics::ShaderRecompiler diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp index e3d945525..14a216d19 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp @@ -81,13 +81,13 @@ void ValidateNativeProgram(const IR::Program& program) { const bool uses_flattened_runtime = !program.srt_reads.empty() || std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) { - return image.indirect_mapping_capacity != 0u; + return image.indirect_search_iterations != 0u; }); if (uses_flattened_runtime) { Expect(Kind::FlattenedSrt); } - if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) { - Expect(Kind::UserData); + if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) { + Expect(Kind::ShaderData); } std::array seen {}; @@ -104,21 +104,19 @@ void ValidateNativeProgram(const IR::Program& program) { Fail(program, "native shader plan is missing a required descriptor group"); } } - const auto has_user_storage = present[static_cast(Kind::UserData)]; - if (program.bindings.push_constant_offset % sizeof(uint32_t) != 0 || - program.bindings.push_constant_offset + program.bindings.push_constant_size > - IR::NativePushConstantSize || + const auto has_shader_data_storage = present[static_cast(Kind::ShaderData)]; + const auto shader_data_dwords = program.bindings.ShaderDataDwords(); + if ((program.bindings.UsesPushData() && + !IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) || program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() || program.bindings.memory_offset_count != program.info.buffers.size() || - (!has_user_storage && - program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) || - (has_user_storage && program.bindings.push_constant_size != 0) || + has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) || !std::is_sorted(program.bindings.user_data_registers.begin(), program.bindings.user_data_registers.end()) || std::adjacent_find(program.bindings.user_data_registers.begin(), program.bindings.user_data_registers.end()) != program.bindings.user_data_registers.end()) { - Fail(program, "native user-data layout is inconsistent"); + Fail(program, "native shader-data layout is inconsistent"); } const auto planning_only_handle = [&](const IR::Inst& handle) { @@ -302,7 +300,6 @@ void AnalyzeProgramRequirements(IR::Program& program) { } std::vector EmitProgram(const IR::Program& program, - const IR::ResourceSnapshot& resources, ShaderStageInputInfo input_info) { using namespace Emitter; @@ -315,15 +312,9 @@ std::vector EmitProgram(const IR::Program& program, !program.spirv_requirements.has_value()) { Fail(program, "SPIR-V emitter requires a fully planned native shader program"); } - if (!IR::ValidateResourceSnapshot(program, resources)) { - Fail(program, "resource snapshot is inconsistent with the native shader plan"); - } - if (!IR::ValidateResourceSpecialization(program, resources)) { - Fail(program, "resource specialization is inconsistent with the native shader plan"); - } ValidateNativeProgram(program); IR::ValidateProgram(program, true); - EmitterState state(program, resources, input_info); + EmitterState state(program, input_info); state.stage = program.stage; state.wave_size = program.wave_size; state.inputs.reserve(program.info.inputs.size()); diff --git a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h index 819b2f2fa..dec536d9e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h +++ b/src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h @@ -3,7 +3,7 @@ #include "common/common.h" #include "common/stringUtils.h" -#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" +#include "graphics/shader/recompiler/ir/ShaderIR.h" #include @@ -12,7 +12,6 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv { void AnalyzeProgramRequirements(IR::Program& program); std::vector EmitProgram(const IR::Program& program, - const IR::ResourceSnapshot& resources, ShaderStageInputInfo input_info); } // namespace Libs::Graphics::ShaderRecompiler::Spirv diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 9790501cb..fe08970fe 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -806,7 +806,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) { return true; } const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg)); - if (state.flattened_srt_variable == 0 || image.indirect_mapping_capacity == 0u || + if (state.flattened_srt_variable == 0 || image.indirect_search_iterations == 0u || image.indirect_resources.size() < 2u) { ctx.Fail(inst, "has no indirect image runtime mapping"); return true; @@ -824,7 +824,7 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) { auto low = ConstantU32(state, 0u); auto high = LoadMapping(mapping); auto selected = ConstantU32(state, 0u); - for (uint32_t iteration = 0; iteration < std::bit_width(image.indirect_mapping_capacity); + for (uint32_t iteration = 0; iteration < image.indirect_search_iterations; iteration++) { const auto active = Binary(state, OpULessThan, TypeBool(state), low, high); const auto mid = diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index bcfc01017..ac05b6ecb 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -341,14 +341,12 @@ constexpr std::array ImageDimensions {{ const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); struct EmitterState { - EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_, - ShaderStageInputInfo input_info_) - : program(program_), resources(resources_), input_info(input_info_), + EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_) + : program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {} Builder builder; const IR::Program& program; - const IR::ResourceSnapshot& resources; ShaderStageInputInfo input_info; const IR::SpirvRequirements& requirements; ShaderType stage = ShaderType::Unknown; @@ -361,7 +359,7 @@ struct EmitterState { uint32_t gds_variable = 0; uint32_t gds_length = 0; uint32_t push_constant_variable = 0; - uint32_t vsharp_storage_variable = 0; + uint32_t shader_data_storage_variable = 0; uint32_t flattened_srt_variable = 0; uint32_t lds_variable = 0; uint32_t scratch_variable = 0; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp index 943035c10..8f7a65a3e 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp @@ -4,6 +4,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { if (state.push_constant_variable != 0) { + dword_index += state.program.bindings.push_data_start_dword; const auto pointer = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer, @@ -12,11 +13,11 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) { state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); return value; } - if (state.vsharp_storage_variable != 0) { + if (state.shader_data_storage_variable != 0) { const auto pointer = state.builder.AllocateId(); const auto value = state.builder.AllocateId(); state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, - state.vsharp_storage_variable, ConstantU32(state, 0), + state.shader_data_storage_variable, ConstantU32(state, 0), ConstantU32(state, dword_index)}); state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); return value; diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp index fb0544b57..e60ba1ca0 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp @@ -119,8 +119,7 @@ uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class) { namespace { uint32_t PushConstantArrayType(EmitterState& state) { - const auto count = - ConstantU32(state, state.program.bindings.push_constant_size / sizeof(uint32_t)); + const auto count = ConstantU32(state, IR::PushData::DwordCount); return state.builder.DecoratedType(OpTypeArray, {TypeU32(state), count}, {{OpDecorate, {DecorationArrayStride, sizeof(uint32_t)}}}); } @@ -128,7 +127,7 @@ uint32_t PushConstantArrayType(EmitterState& state) { uint32_t PushConstantBlockType(EmitterState& state) { return state.builder.DecoratedType( OpTypeStruct, {PushConstantArrayType(state)}, - {{OpMemberDecorate, {0, DecorationOffset, state.program.bindings.push_constant_offset}}, + {{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}}); } @@ -173,14 +172,14 @@ void DefineDescriptorVariables(EmitterState& state) { state.fault_buffer_variable = state.builder.DefineGlobalVariable( TypeStorageBufferPointer(state), StorageClassStorageBuffer); } - if (state.program.bindings.push_constant_size != 0) { + if (state.program.bindings.UsesPushData()) { const auto pointer_type = TypePointer(state, StorageClassPushConstant, PushConstantBlockType(state)); state.push_constant_variable = state.builder.DefineGlobalVariable(pointer_type, StorageClassPushConstant); } - if (DescriptorBinding(state, IR::DescriptorBindingKind::UserData) != nullptr) { - state.vsharp_storage_variable = state.builder.DefineGlobalVariable( + if (DescriptorBinding(state, IR::DescriptorBindingKind::ShaderData) != nullptr) { + state.shader_data_storage_variable = state.builder.DefineGlobalVariable( TypeStorageBufferPointer(state), StorageClassStorageBuffer); } if (DescriptorBinding(state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) { @@ -604,9 +603,9 @@ void AddVsharpAnnotationsAndNames(EmitterState& state) { state.builder.AddName(PushConstantBlockType(state), "BufferResource"); state.builder.AddName(state.push_constant_variable, "vsharp"); } - if (state.vsharp_storage_variable != 0) { - DecorateDescriptor(state, state.vsharp_storage_variable, "user_data", - IR::DescriptorBindingKind::UserData); + if (state.shader_data_storage_variable != 0) { + DecorateDescriptor(state, state.shader_data_storage_variable, "shader_data", + IR::DescriptorBindingKind::ShaderData); } } diff --git a/src/graphics/shader/recompiler/ir/Program.cpp b/src/graphics/shader/recompiler/ir/Program.cpp index f1445e017..ef71877b5 100644 --- a/src/graphics/shader/recompiler/ir/Program.cpp +++ b/src/graphics/shader/recompiler/ir/Program.cpp @@ -54,7 +54,7 @@ bool IsRuntimeRead(ValueOpcode opcode) { AddressOpcodeInfoOf(opcode).access == AddressAccess::Read; } -bool EquivalentValue(const Program& program, Value left, Value right, +bool EquivalentValue(const ResourcePlan& program, Value left, Value right, std::vector>& visited) { left = left.Resolve(); right = right.Resolve(); @@ -97,6 +97,20 @@ bool EquivalentValue(const Program& program, Value left, Value right, } // namespace +ResourcePlan::~ResourcePlan() { + for (auto& inst: value_storage) { + inst.Invalidate(); + } +} + +ResourcePlan& ResourcePlan::operator=(ResourcePlan&& other) noexcept { + if (this != &other) { + this->~ResourcePlan(); + new (this) ResourcePlan(std::move(other)); + } + return *this; +} + Program::~Program() { // Values may cross block boundaries. Detach all arguments before any block starts destroying // its instruction storage so reverse-use links always point to live definitions. @@ -115,12 +129,31 @@ Program& Program::operator=(Program&& other) noexcept { return *this; } -bool EquivalentValue(const Program& program, Value left, Value right) { +CompiledShaderInfo Program::TakeCompiledInfo() && { + CompiledShaderInfo result { + .stage = stage, + .shader_hash = shader_hash, + .wave_size = wave_size, + .user_data_base = user_data_base, + .user_data_count = user_data_count, + .scratch_dwords = scratch_dwords, + .info = std::move(info), + .bindings = std::move(bindings), + }; + for (const auto& output: result.info.outputs) { + if (output.kind == StageOutputKind::Parameter && output.index < 32) { + result.param_export_mask |= 1u << output.index; + } + } + return result; +} + +bool EquivalentValue(const ResourcePlan& program, Value left, Value right) { std::vector> visited; return EquivalentValue(program, left, right, visited); } -Value ResolveInvariantPhi(const Program& program, Value value) { +Value ResolveInvariantPhi(const ResourcePlan& program, Value value) { value = value.Resolve(); const auto* root = value.TryInstruction(); if (root == nullptr || root->GetOpcode() != ValueOpcode::Phi) { diff --git a/src/graphics/shader/recompiler/ir/ResourceSnapshot.h b/src/graphics/shader/recompiler/ir/ResourceSnapshot.h new file mode 100644 index 000000000..b74ea9d4e --- /dev/null +++ b/src/graphics/shader/recompiler/ir/ResourceSnapshot.h @@ -0,0 +1,29 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCESNAPSHOT_H_ +#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCESNAPSHOT_H_ + +#include +#include +#include + +namespace Libs::Graphics::ShaderRecompiler::IR { + +struct DescriptorValue { + std::array dwords = {}; + uint32_t dword_count = 0; + + bool operator==(const DescriptorValue& other) const { + return dword_count == other.dword_count && dwords == other.dwords; + } +}; + +struct ResourceSnapshot { + std::vector buffers; + std::vector images; + std::vector samplers; + std::vector flattened_srt; + std::vector user_data; +}; + +} // namespace Libs::Graphics::ShaderRecompiler::IR + +#endif // EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCESNAPSHOT_H_ diff --git a/src/graphics/shader/recompiler/ir/ShaderIR.h b/src/graphics/shader/recompiler/ir/ShaderIR.h index a910ed559..d50a0e2b7 100644 --- a/src/graphics/shader/recompiler/ir/ShaderIR.h +++ b/src/graphics/shader/recompiler/ir/ShaderIR.h @@ -8,11 +8,13 @@ #include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" #include "graphics/shader/recompiler/ir/Block.h" +#include "graphics/shader/recompiler/ir/ResourceSnapshot.h" #include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.h" #include "graphics/shader/shader.h" #include #include +#include #include #include #include @@ -87,15 +89,6 @@ struct ExportInfo { bool operator==(const ExportInfo& other) const = default; }; -struct DescriptorValue { - std::array dwords = {}; - uint32_t dword_count = 0; - - bool operator==(const DescriptorValue& other) const { - return dword_count == other.dword_count && dwords == other.dwords; - } -}; - struct BufferResource { static constexpr uint32_t NoImageAlias = UINT32_MAX; @@ -139,7 +132,7 @@ struct ImageResource { bool r128 = false; uint32_t indirect_root = NoIndirectImage; uint32_t indirect_mapping_offset = 0; - uint32_t indirect_mapping_capacity = 0; + uint32_t indirect_search_iterations = 0; std::vector indirect_resources; bool operator==(const ImageResource& other) const = default; @@ -149,6 +142,7 @@ struct SamplerResource { uint32_t source = 0; uint32_t first_use_pc = 0; bool force_point_filtering = false; + bool depth_compare = false; bool operator==(const SamplerResource& other) const = default; }; @@ -270,14 +264,28 @@ enum class DescriptorBindingKind : uint32_t { BdaPagetable, FaultBuffer, FlattenedSrt, - UserData, + ShaderData, Count, }; static_assert(static_cast(DescriptorBindingKind::Samplers) == 37u); static_assert(static_cast(DescriptorBindingKind::Count) == 43u); -constexpr uint32_t NativePushConstantSize = 128; +struct PushData { + static constexpr uint32_t DwordCount = 32; + static constexpr uint32_t NoStart = UINT32_MAX; + std::array dwords {}; + + [[nodiscard]] static constexpr bool CanFit(uint32_t start, uint32_t size) { + return size != 0 && start <= DwordCount && size <= DwordCount - start; + } + [[nodiscard]] static constexpr uint32_t StartFor(uint32_t cursor, uint32_t size) { + return CanFit(cursor, size) ? cursor : NoStart; + } +}; + +static_assert(sizeof(PushData) == 128); +constexpr uint32_t NativePushConstantSize = sizeof(PushData); [[nodiscard]] constexpr uint32_t NativeBinding(ShaderType stage, DescriptorBindingKind kind) { return static_cast(kind) + @@ -375,8 +383,7 @@ struct DescriptorBinding { }; struct BindingLayout { - uint32_t push_constant_offset = 0; - uint32_t push_constant_size = 0; + uint32_t push_data_start_dword = PushData::NoStart; uint32_t memory_offset_dword = 0; uint32_t memory_offset_count = 0; std::vector user_data_registers; @@ -385,6 +392,14 @@ struct BindingLayout { [[nodiscard]] uint32_t ShaderDataDwords() const { return memory_offset_dword + (memory_offset_count + 3u) / 4u; } + [[nodiscard]] bool UsesPushData() const { + return push_data_start_dword != PushData::NoStart; + } + void AdvancePushData(uint32_t& cursor) const { + if (UsesPushData()) { + cursor = push_data_start_dword + ShaderDataDwords(); + } + } bool operator==(const BindingLayout& other) const = default; }; @@ -454,7 +469,47 @@ struct SrtRead { bool operator==(const SrtRead& other) const = default; }; -struct Program { +// Stable shader metadata consumed by the renderer after native IR has been discarded. +struct CompiledShaderInfo { + ShaderType stage = ShaderType::Unknown; + uint64_t shader_hash = 0; + uint32_t wave_size = 64; + uint32_t user_data_base = 0; + uint32_t user_data_count = 64; + uint32_t scratch_dwords = 0; + uint32_t param_export_mask = 0; + ShaderInfo info; + BindingLayout bindings; +}; + +// Immutable runtime resource analysis retained by the shader cache. It owns only the native +// value graph reachable from descriptors/SRT reads, rather than the translated shader CFG. +struct ResourcePlan { + ResourcePlan() = default; + ~ResourcePlan(); + + ResourcePlan(const ResourcePlan&) = delete; + ResourcePlan& operator=(const ResourcePlan&) = delete; + ResourcePlan(ResourcePlan&&) noexcept = default; + ResourcePlan& operator=(ResourcePlan&& other) noexcept; + + ShaderType stage = ShaderType::Unknown; + uint64_t shader_hash = 0; + uint32_t user_data_base = 0; + uint32_t user_data_count = 64; + std::list value_storage; + std::vector memory_info; + std::vector descriptor_sources; + std::vector materialization_sources; + std::vector srt_reads; + std::vector clean_flat_slots; + bool requires_specialization_memory = false; + bool srt_plan_complete = false; + bool resource_tracking_complete = false; + ShaderInfo info; +}; + +struct Program: ResourcePlan { Program() = default; ~Program(); @@ -462,31 +517,22 @@ struct Program { Program& operator=(const Program&) = delete; Program(Program&&) noexcept = default; Program& operator=(Program&& other) noexcept; + CompiledShaderInfo TakeCompiledInfo() &&; - ShaderType stage = ShaderType::Unknown; - uint64_t shader_hash = 0; - uint32_t wave_size = 64; - uint32_t user_data_base = 0; - uint32_t user_data_count = 64; - uint32_t scratch_dwords = 0; + std::vector> block_storage; + BlockList blocks; + uint32_t wave_size = 64; + uint32_t scratch_dwords = 0; bool dispatcher_fallback = false; CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; std::string fallback_reason; - std::vector> block_storage; - BlockList blocks; - std::vector block_info; + std::vector block_info; // Decoded MIMG/VMEM metadata carries details such as RDNA2 NSA address registers and // storage-image swizzles. Typed memory instructions carry a dense index into these shader-local // tables until those fields are consumed by emission. - std::vector memory_info; std::vector export_info; - std::vector descriptor_sources; - std::vector srt_reads; std::vector dynamic_reads; - bool srt_plan_complete = false; - ShaderInfo info; - bool resource_tracking_complete = false; - bool shader_info_complete = false; + bool shader_info_complete = false; BindingLayout bindings; bool binding_layout_complete = false; @@ -497,8 +543,8 @@ std::string ProgramToString(const Program& program); void ValidateProgram(const Program& program, bool require_ssa); void ResolveControlFlowIdentities(Program& program); -bool EquivalentValue(const Program& program, Value left, Value right); -Value ResolveInvariantPhi(const Program& program, Value value); +bool EquivalentValue(const ResourcePlan& program, Value left, Value right); +Value ResolveInvariantPhi(const ResourcePlan& program, Value value); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp index 7a4608ca8..0c341f70a 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.cpp @@ -69,24 +69,18 @@ bool UsesGds(const Program& program) { } // namespace -void AllocateBindings(Program& program, uint32_t push_constant_offset) { +void AllocateBindings(Program& program, uint32_t push_data_start_dword) { if (!program.shader_info_complete || program.binding_layout_complete) { EXIT("shader binding layout failed: %s", !program.shader_info_complete ? "shader info is not ready" : "binding layout already allocated"); } - if (push_constant_offset > NativePushConstantSize || - push_constant_offset % sizeof(uint32_t) != 0) { - EXIT("shader binding layout failed: push-constant offset %u exceeds the Vulkan minimum " - "guarantee or is unaligned", - push_constant_offset); - } - BindingLayout next; - next.push_constant_offset = push_constant_offset; next.user_data_registers = CollectUserData(program); next.memory_offset_dword = static_cast(next.user_data_registers.size()); next.memory_offset_count = static_cast(program.info.buffers.size()); + next.push_data_start_dword = + PushData::StartFor(push_data_start_dword, next.ShaderDataDwords()); if (!program.info.buffers.empty()) { std::vector resources(program.info.buffers.size()); @@ -139,17 +133,14 @@ void AllocateBindings(Program& program, uint32_t push_constant_offset) { const bool uses_flattened_runtime = !program.srt_reads.empty() || std::ranges::any_of(program.info.images, [](const ImageResource& image) { - return image.indirect_mapping_capacity != 0u; + return image.indirect_search_iterations != 0u; }); if (uses_flattened_runtime) { AddBinding(next, DescriptorBindingKind::FlattenedSrt); } - const auto push_dwords = (NativePushConstantSize - push_constant_offset) / sizeof(uint32_t); - if (next.ShaderDataDwords() <= push_dwords) { - next.push_constant_size = next.ShaderDataDwords() * sizeof(uint32_t); - } else { - AddBinding(next, DescriptorBindingKind::UserData); + if (next.ShaderDataDwords() != 0 && !next.UsesPushData()) { + AddBinding(next, DescriptorBindingKind::ShaderData); } program.bindings = std::move(next); diff --git a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h index 8ac25cd3d..9999c4438 100644 --- a/src/graphics/shader/recompiler/ir/passes/BindingLayout.h +++ b/src/graphics/shader/recompiler/ir/passes/BindingLayout.h @@ -5,7 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -void AllocateBindings(Program& program, uint32_t push_constant_offset); +void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0); const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind); diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 11a494c30..ba2df4156 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -19,9 +21,22 @@ namespace { constexpr uint64_t AddressMask = 0x0000ffffffffffffull; constexpr uint64_t MaxIndirectImageProbes = 65536u; -[[noreturn]] void SpecializationFail(std::string_view message) { - EXIT("shader resource specialization failed: %s", std::string(message).c_str()); - std::abort(); +struct IndirectImage { + uint32_t resource = 0; + std::vector keys; + std::vector candidates; + std::vector descriptors; +}; + +struct MaterializedSnapshot { + ResourceSnapshot resources; + std::vector indirect_images; +}; + +bool SpecializationFail(std::string_view message) { + std::fprintf(stderr, "shader resource specialization failed: %.*s\n", + static_cast(message.size()), message.data()); + return false; } Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor, @@ -94,6 +109,11 @@ bool RequiresPointSampler(const ImageResource& image) { image.conversion_format != Prospero::BufferFormat::kInvalid; } +bool RequiresPointSampler(const ResourceSpecialization::Image& image) { + return image.numeric_class == Prospero::TextureNumericClass::Sint || + image.conversion_format != Prospero::BufferFormat::kInvalid; +} + bool DescriptorIsCube(const DescriptorValue& descriptor) { return static_cast((descriptor.dwords[3] >> 28u) & 0xfu) == Prospero::ImageType::kCube; @@ -116,14 +136,14 @@ bool DecodeBufferDescriptor(const DescriptorValue& descriptor, ShaderBufferResou return true; } -const DescriptorSource* Source(const Program& program, uint32_t source) { +const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) { if (source >= program.descriptor_sources.size()) { return nullptr; } return &program.descriptor_sources[source]; } -void MarkCleanFlatSlots(const Program& program, const DescriptorSource* source, +void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* source, std::span slots) { if (source == nullptr) { return; @@ -186,8 +206,8 @@ bool ReadScalarBufferWord(const ShaderBufferResource& descriptor, uint32_t dynam bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, const DescriptorValue& material_value, - const DescriptorValue& heap_value, const SrtRuntime& runtime, - ResourceSnapshot::IndirectImage& result) { + const DescriptorValue& heap_value, bool r128, + const SrtRuntime& runtime, IndirectImage& result) { ShaderBufferResource material; ShaderBufferResource heap; if (!DecodeBufferDescriptor(material_value, material) || @@ -212,6 +232,8 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, std::vector keys {0u}; std::unordered_set seen {0u}; + keys.reserve(static_cast(probe_count) + 1u); + seen.reserve(static_cast(probe_count) + 1u); for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) { uint32_t key = 0; if (!ReadScalarBufferWord(material, static_cast(offset), 0u, runtime, key)) { @@ -225,9 +247,11 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, } } - ResourceSnapshot::IndirectImage next; - next.capacity = static_cast(probe_count + 1u); - next.keys = std::move(keys); + IndirectImage next; + next.keys = std::move(keys); + next.candidates.reserve(next.keys.size()); + next.descriptors.reserve( + std::min(next.keys.size(), static_cast(ShaderInfo::MaxImages))); for (const auto key: next.keys) { DescriptorValue candidate; candidate.dword_count = 8u; @@ -238,11 +262,14 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, return false; } } - if (NullImageDescriptor(candidate) || !ValidImageDescriptor(candidate)) { + if (NullImageDescriptor(candidate) || !ValidImageDescriptor(candidate, r128)) { candidate.dwords.fill(0); } const auto found = std::ranges::find(next.descriptors, candidate); if (found == next.descriptors.end()) { + if (next.descriptors.size() >= ShaderInfo::MaxImages) { + return false; + } next.descriptors.push_back(candidate); next.candidates.push_back(static_cast(next.descriptors.size() - 1u)); } else { @@ -253,339 +280,122 @@ bool MaterializeIndirectImage(const DescriptorSource::IndirectImage& indirect, return true; } -size_t FlattenedRuntimeDwords(const Program& program) { - size_t size = program.srt_reads.size(); - for (uint32_t resource = 0; resource < program.info.images.size(); resource++) { - const auto& image = program.info.images[resource]; - if (image.indirect_root == resource) { - size = std::max(size, static_cast(image.indirect_mapping_offset) + 1u + - static_cast(image.indirect_mapping_capacity) * 2u); - } - } - return size; -} - } // namespace -bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot) { - if (!program.resource_tracking_complete) { - return false; - } - if (snapshot.buffers.size() != program.info.buffers.size() || - snapshot.images.size() != program.info.images.size() || - snapshot.samplers.size() != program.info.samplers.size()) { - return false; - } - if (snapshot.flattened_srt.size() != FlattenedRuntimeDwords(program)) { - return false; - } - if (program.binding_layout_complete) { - for (const auto reg: program.bindings.user_data_registers) { - if (reg < program.user_data_base || - reg - program.user_data_base >= snapshot.user_data.size()) { - return false; - } - } - } - for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - const auto alias = program.info.buffers[i].image_alias; - if (alias != BufferResource::NoImageAlias && alias >= program.info.images.size()) { - return false; - } - } - const auto CheckWidth = [&](const auto& values, uint32_t width) { - for (uint32_t i = 0; i < values.size(); i++) { - if (values[i].dword_count != width) { - return false; - } - } - return true; - }; - std::unordered_set indirect_resources; - for (const auto& table: snapshot.indirect_images) { - const auto* source = table.resource < program.info.images.size() - ? Source(program, program.info.images[table.resource].source) - : nullptr; - if (source == nullptr || !source->indirect_image.has_value() || - program.info.images[table.resource].indirect_root != ImageResource::NoIndirectImage || - table.keys.empty() || table.keys.size() != table.candidates.size() || - table.capacity < table.keys.size() || table.descriptors.empty() || - !indirect_resources.insert(table.resource).second) { - return false; - } - for (const auto candidate: table.candidates) { - if (candidate >= table.descriptors.size()) { - return false; - } - } - if (snapshot.images[table.resource] != table.descriptors[table.candidates[0]]) { - return false; - } - } - return CheckWidth(snapshot.buffers, 4) && CheckWidth(snapshot.images, 8) && - CheckWidth(snapshot.samplers, 4); -} - -bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot) { - if (!ValidateResourceSnapshot(program, snapshot)) { - return false; - } - if (!snapshot.indirect_images.empty()) { - return false; - } - for (uint32_t i = 0; i < program.info.buffers.size(); i++) { - const auto& buffer = program.info.buffers[i]; - ShaderBufferResource descriptor; - if (!DecodeBufferDescriptor(snapshot.buffers[i], descriptor)) { - return false; - } - if (buffer.packed_stride != descriptor.PackedStride() || - buffer.descriptor_format != descriptor.Format() || - buffer.descriptor_swizzle != descriptor.DstSelXYZW()) { - return false; - } - } - for (uint32_t i = 0; i < program.info.images.size(); i++) { - const auto& image = program.info.images[i]; - const auto& descriptor = snapshot.images[i]; - const auto mip_count = StorageMipCount(image, descriptor); - if (mip_count == 0u || mip_count != image.mip_count) { - return false; - } - if (NullImageDescriptor(descriptor)) { - if (image.indirect_root != ImageResource::NoIndirectImage) { - const auto root = image.indirect_root; - if (root >= program.info.images.size() || - program.info.images[root].resource_class != image.resource_class || - program.info.images[root].numeric_class != image.numeric_class || - program.info.images[root].dimension != image.dimension || - program.info.images[root].mip_mode != image.mip_mode || - program.info.images[root].mip_count != image.mip_count || - program.info.images[root].conversion_format != image.conversion_format || - program.info.images[root].shader_swizzle != image.shader_swizzle || - program.info.images[root].cube != image.cube) { - return false; - } - continue; - } - const auto canonical_numeric = image.atomic ? Prospero::TextureNumericClass::Uint - : Prospero::TextureNumericClass::Float; - if (image.dimension != Decoder::ImageDimension::Dim2D || image.cube || - image.resource_class == ImageResourceClass::None || - image.numeric_class != canonical_numeric) { - return false; - } - continue; - } - const auto dimension = DescriptorDimension(descriptor, image.dimension); - if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension || - DescriptorIsCube(descriptor) != image.cube) { - return false; - } - if (image.resource_class == ImageResourceClass::None) { - return false; - } - const bool storage = image.resource_class == ImageResourceClass::Storage; - const auto format = - static_cast((descriptor.dwords[1] >> 20u) & 0x1ffu); - if (image.atomic && format != Prospero::BufferFormat::k32UInt) { - return false; - } - const auto conversion_format = ImageConversionFormat(format); - if (image.conversion_format != conversion_format) { - return false; - } - if ((storage || conversion_format != Prospero::BufferFormat::kInvalid) && - image.shader_swizzle != DescriptorImageSwizzle(descriptor)) { - return false; - } - auto numeric_class = Prospero::SampledTextureNumericClass(format); - if (storage) { - const bool raw_sint = format == Prospero::BufferFormat::k32SInt && image.written && - !image.read && !image.atomic; - if ((!raw_sint && numeric_class == Prospero::TextureNumericClass::Sint) || - numeric_class == Prospero::TextureNumericClass::Unsupported) { - return false; - } - if (raw_sint) { - numeric_class = Prospero::TextureNumericClass::Uint; - } - } else if (image.depth_compare && numeric_class != Prospero::TextureNumericClass::Float) { - return false; - } - if (image.numeric_class != numeric_class) { - return false; - } - } - return true; -} - -bool MaterializeResources(const Program& program, const SrtRuntime& runtime, - ResourceSnapshot& snapshot) { +static bool MaterializeSnapshot(const ResourcePlan& program, const SrtRuntime& runtime, + MaterializedSnapshot& snapshot) { if (!program.resource_tracking_complete) { return false; } - std::vector requests; - std::vector clean_flat_slots(program.srt_reads.size()); - requests.reserve(program.info.buffers.size() + program.info.images.size() * 2u + - program.info.samplers.size()); - for (const auto& buffer: program.info.buffers) { - requests.push_back({buffer.source}); - } - for (const auto& image: program.info.images) { - const auto* source = Source(program, image.source); - if (source != nullptr && source->indirect_image.has_value()) { - if (runtime.read_specialization_memory == nullptr) { - return false; - } - MarkCleanFlatSlots(program, Source(program, source->indirect_image->material_source), - clean_flat_slots); - MarkCleanFlatSlots(program, Source(program, source->indirect_image->heap_source), - clean_flat_slots); - } else { - requests.push_back({image.source}); - } - } - for (const auto& sampler: program.info.samplers) { - requests.push_back({sampler.source}); + if (program.requires_specialization_memory && runtime.read_specialization_memory == nullptr) { + return false; } std::vector values; std::vector flattened_srt; - if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, - clean_flat_slots)) { + if (!EvaluateRuntimeSources(program, program.materialization_sources, runtime, values, + flattened_srt, program.clean_flat_slots)) { return false; } - ResourceSnapshot next; - auto cursor = values.begin(); + auto& next = snapshot.resources; + auto cursor = values.begin(); next.buffers.assign(cursor, cursor + program.info.buffers.size()); cursor += program.info.buffers.size(); - for (auto& descriptor: next.buffers) { - ShaderBufferResource buffer; - if (DecodeBufferDescriptor(descriptor, buffer) && buffer.Type() != 0) { - descriptor.dwords.fill(0); - } - } next.flattened_srt = std::move(flattened_srt); - next.flattened_srt.resize(FlattenedRuntimeDwords(program)); next.images.resize(program.info.images.size()); - std::vector image_written(program.info.images.size()); for (uint32_t image_index = 0; image_index < program.info.images.size(); image_index++) { const auto& image = program.info.images[image_index]; const auto* source = Source(program, image.source); if (source != nullptr && source->indirect_image.has_value()) { - if (image.indirect_root != ImageResource::NoIndirectImage && - image.indirect_root != image_index) { - continue; - } - const std::array requests { - DescriptorSourceRequest {source->indirect_image->material_source}, - DescriptorSourceRequest {source->indirect_image->heap_source}}; - SrtRuntime clean_runtime = runtime; - clean_runtime.read_memory = runtime.read_specialization_memory; + const std::array requests {source->indirect_image->material_source, + source->indirect_image->heap_source}; + SrtRuntime clean_runtime = runtime; + clean_runtime.read_memory = runtime.read_specialization_memory; std::vector tables; if (!EvaluateDescriptorSources(program, requests, clean_runtime, tables)) { return false; } - const auto& material = tables[0]; - const auto& heap = tables[1]; - ResourceSnapshot::IndirectImage table; - if (!MaterializeIndirectImage(*source->indirect_image, material, heap, runtime, - table)) { + const auto& material = tables[0]; + const auto& heap = tables[1]; + IndirectImage table; + if (!MaterializeIndirectImage(*source->indirect_image, material, heap, image.r128, + runtime, table)) { return false; } - if (image.indirect_root == ImageResource::NoIndirectImage) { - next.images[image_index] = table.descriptors[table.candidates[0]]; - image_written[image_index] = 1u; - if (table.descriptors.size() > 1u) { - table.resource = image_index; - next.indirect_images.push_back(std::move(table)); - } - continue; - } - if (table.keys.size() > image.indirect_mapping_capacity || - table.descriptors.size() > image.indirect_resources.size()) { - return false; - } - for (uint32_t candidate = 0; candidate < image.indirect_resources.size(); candidate++) { - const auto resource = image.indirect_resources[candidate]; - if (resource >= program.info.images.size() || - program.info.images[resource].indirect_root != image_index) { - return false; - } - next.images[resource] = - table.descriptors[candidate < table.descriptors.size() ? candidate : 0u]; - image_written[resource] = 1u; - } - std::vector order(table.keys.size()); - std::iota(order.begin(), order.end(), 0u); - std::ranges::sort(order, {}, [&](uint32_t index) { return table.keys[index]; }); - const auto mapping = image.indirect_mapping_offset; - next.flattened_srt[mapping] = static_cast(table.keys.size()); - for (uint32_t entry = 0; entry < order.size(); entry++) { - const auto source = order[entry]; - const auto offset = mapping + 1u + entry * 2u; - next.flattened_srt[offset] = table.keys[source]; - next.flattened_srt[offset + 1] = table.candidates[source]; + next.images[image_index] = table.descriptors[table.candidates[0]]; + if (table.descriptors.size() > 1u) { + table.resource = image_index; + snapshot.indirect_images.push_back(std::move(table)); } } else { auto descriptor = *cursor++; if (!ValidImageDescriptor(descriptor, image.r128)) { descriptor.dwords.fill(0); } - next.images[image_index] = descriptor; - image_written[image_index] = 1u; + next.images[image_index] = descriptor; } } - if (std::ranges::find(image_written, uint8_t {0}) != image_written.end()) { - return false; - } next.samplers.assign(cursor, cursor + program.info.samplers.size()); next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end()); - if (!ValidateResourceSnapshot(program, next)) { - return false; - } - snapshot = std::move(next); return true; } -void SpecializeResources(Program& program, ResourceSnapshot& snapshot) { - if (!program.resource_tracking_complete || program.shader_info_complete || - program.binding_layout_complete) { - return SpecializationFail(!program.resource_tracking_complete - ? "shader resources were not tracked" - : "resource specialization is too late"); - } - if (!ValidateResourceSnapshot(program, snapshot)) { - return SpecializationFail("invalid resource snapshot"); - } +struct SamplerPlan { + std::array point_sampler {}; + uint32_t sampler_count = 0; +}; - auto next = program.info; - auto next_snapshot = snapshot; +template +bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan); + +static bool BuildResourceSpecialization(const ResourcePlan& program, MaterializedSnapshot snapshot, + ResourceSnapshot& specialized_snapshot, + ResourceSpecialization& specialization) { + auto next_snapshot = std::move(snapshot.resources); + ResourceSpecialization next_specialization; + next_specialization.buffers.reserve(program.info.buffers.size()); + size_t image_count = program.info.images.size(); + size_t mapping_words = 0; for (const auto& table: snapshot.indirect_images) { - if (table.resource >= next.images.size() || table.descriptors.size() < 2u || - next.images.size() + table.descriptors.size() - 1u > ShaderInfo::MaxImages) { + if (table.resource >= program.info.images.size() || table.descriptors.size() < 2u || + image_count + table.descriptors.size() - 1u > ShaderInfo::MaxImages) { return SpecializationFail( "indirect image candidates exceed the dense image resource limit"); } - const auto root_image = next.images[table.resource]; - std::vector resources(table.descriptors.size()); - resources[0] = table.resource; + image_count += table.descriptors.size() - 1u; + mapping_words += 1u + table.keys.size() * 2u; + } + next_snapshot.images.reserve(image_count); + next_snapshot.flattened_srt.reserve(next_snapshot.flattened_srt.size() + mapping_words); + next_specialization.images.reserve(image_count); + for (const auto& image: program.info.images) { + next_specialization.images.push_back({ + .numeric_class = image.numeric_class, + .dimension = image.dimension, + .mip_count = image.mip_count, + .conversion_format = image.conversion_format, + .shader_swizzle = image.shader_swizzle, + .indirect_root = image.indirect_root, + .indirect_mapping_offset = image.indirect_mapping_offset, + .indirect_search_iterations = image.indirect_search_iterations, + .cube = image.cube, + }); + } + for (const auto& table: snapshot.indirect_images) { + const auto root_image = next_specialization.images[table.resource]; for (uint32_t candidate = 1; candidate < table.descriptors.size(); candidate++) { - resources[candidate] = static_cast(next.images.size()); - auto image = root_image; - image.indirect_root = table.resource; - next.images.push_back(std::move(image)); + auto image = root_image; + image.indirect_root = table.resource; + next_specialization.images.push_back(image); next_snapshot.images.push_back(table.descriptors[candidate]); } - auto& root = next.images[table.resource]; - root.indirect_root = table.resource; - root.indirect_mapping_offset = static_cast(next_snapshot.flattened_srt.size()); - root.indirect_mapping_capacity = table.capacity; - root.indirect_resources = std::move(resources); + auto& root = next_specialization.images[table.resource]; + root.indirect_root = table.resource; + root.indirect_mapping_offset = static_cast(next_snapshot.flattened_srt.size()); + root.indirect_search_iterations = std::bit_width(table.keys.size()); next_snapshot.flattened_srt.resize(next_snapshot.flattened_srt.size() + 1u + - static_cast(table.capacity) * 2u); + table.keys.size() * 2u); std::vector order(table.keys.size()); std::iota(order.begin(), order.end(), 0u); std::ranges::sort(order, {}, [&](uint32_t index) { return table.keys[index]; }); @@ -599,36 +409,58 @@ void SpecializeResources(Program& program, ResourceSnapshot& snapshot) { } next_snapshot.images[table.resource] = table.descriptors[0]; } - next_snapshot.indirect_images.clear(); - for (uint32_t i = 0; i < next.buffers.size(); i++) { + for (uint32_t i = 0; i < program.info.buffers.size(); i++) { + auto& descriptor_value = next_snapshot.buffers[i]; ShaderBufferResource descriptor; - if (!DecodeBufferDescriptor(next_snapshot.buffers[i], descriptor)) { + if (!DecodeBufferDescriptor(descriptor_value, descriptor)) { return SpecializationFail(fmt::format("buffer descriptor {} has invalid width", i)); } - next.buffers[i].packed_stride = descriptor.PackedStride(); - next.buffers[i].descriptor_format = descriptor.Format(); - next.buffers[i].descriptor_swizzle = descriptor.DstSelXYZW(); + if (descriptor.Type() != 0) { + descriptor_value.dwords.fill(0); + descriptor = {}; + } + auto packed_stride = descriptor.PackedStride(); + const auto stride = packed_stride & 0x3fffu; + const bool swizzle = stride != 0u && ((packed_stride >> 14u) & 1u) != 0u; + if (stride == 0u) { + packed_stride &= ~((1u << 14u) | (3u << 16u)); + } else if (!swizzle) { + packed_stride &= ~(3u << 16u); + } + next_specialization.buffers.push_back({ + .packed_stride = packed_stride, + .descriptor_format = program.info.buffers[i].formatted + ? descriptor.Format() + : Prospero::BufferFormat::kInvalid, + .descriptor_swizzle = + program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7), + }); } - for (uint32_t i = 0; i < next.images.size(); i++) { + for (uint32_t i = 0; i < next_specialization.images.size(); i++) { const auto& descriptor = next_snapshot.images[i]; - auto& image = next.images[i]; - if (image.resource_class == ImageResourceClass::None || - (image.atomic && image.resource_class != ImageResourceClass::Storage)) { + auto& image = next_specialization.images[i]; + const auto base_index = i < program.info.images.size() ? i : image.indirect_root; + if (base_index >= program.info.images.size()) { + return SpecializationFail(fmt::format("image resource {} has an invalid root", i)); + } + const auto& base = program.info.images[base_index]; + if (base.resource_class == ImageResourceClass::None || + (base.atomic && base.resource_class != ImageResourceClass::Storage)) { return SpecializationFail(fmt::format("image resource {} has an invalid class", i)); } - image.mip_count = StorageMipCount(image, descriptor); + image.mip_count = StorageMipCount(base, descriptor); if (image.mip_count == 0u) { return SpecializationFail( fmt::format("storage image descriptor {} has an invalid mip range", i)); } if (NullImageDescriptor(descriptor)) { - image.numeric_class = image.atomic ? Prospero::TextureNumericClass::Uint - : Prospero::TextureNumericClass::Float; + image.numeric_class = base.atomic ? Prospero::TextureNumericClass::Uint + : Prospero::TextureNumericClass::Float; image.dimension = Decoder::ImageDimension::Dim2D; image.cube = false; continue; } - const auto descriptor_dimension = DescriptorDimension(descriptor, image.dimension); + const auto descriptor_dimension = DescriptorDimension(descriptor, base.dimension); if (descriptor_dimension == Decoder::ImageDimension::Unknown) { return SpecializationFail(fmt::format( "image descriptor {} has unsupported type {}: {:08x},{:08x},{:08x},{:08x}," @@ -641,19 +473,19 @@ void SpecializeResources(Program& program, ResourceSnapshot& snapshot) { image.cube = DescriptorIsCube(descriptor); const auto format = static_cast((descriptor.dwords[1] >> 20u) & 0x1ffu); - if (image.atomic && format != Prospero::BufferFormat::k32UInt) { + if (base.atomic && format != Prospero::BufferFormat::k32UInt) { return SpecializationFail( fmt::format("atomic image descriptor {} uses unsupported format {}", i, static_cast(format))); } - const bool storage = image.resource_class == ImageResourceClass::Storage; + const bool storage = base.resource_class == ImageResourceClass::Storage; image.conversion_format = ImageConversionFormat(format); if (storage || image.conversion_format != Prospero::BufferFormat::kInvalid) { image.shader_swizzle = DescriptorImageSwizzle(descriptor); } const bool raw_sint_storage = storage && format == Prospero::BufferFormat::k32SInt && - image.written && !image.read && !image.atomic; - image.numeric_class = Prospero::SampledTextureNumericClass(format); + base.written && !base.read && !base.atomic; + image.numeric_class = Prospero::SampledTextureNumericClass(format); if (storage) { if ((!raw_sint_storage && image.numeric_class == Prospero::TextureNumericClass::Sint) || image.numeric_class == Prospero::TextureNumericClass::Unsupported) { @@ -665,46 +497,49 @@ void SpecializeResources(Program& program, ResourceSnapshot& snapshot) { image.numeric_class = Prospero::TextureNumericClass::Uint; } } else if (image.numeric_class == Prospero::TextureNumericClass::Unsupported || - (image.depth_compare && + (base.depth_compare && image.numeric_class != Prospero::TextureNumericClass::Float)) { return SpecializationFail( fmt::format("sampled image descriptor {} uses unsupported format {}", i, static_cast(format))); } } - for (uint32_t root_index = 0; root_index < next.images.size(); root_index++) { - auto& root = next.images[root_index]; + for (uint32_t root_index = 0; root_index < next_specialization.images.size(); root_index++) { + auto& root = next_specialization.images[root_index]; if (root.indirect_root != root_index) { continue; } - if (root.indirect_mapping_capacity == 0u || root.indirect_resources.size() < 2u || + const auto key_count = root.indirect_mapping_offset < next_snapshot.flattened_srt.size() + ? next_snapshot.flattened_srt[root.indirect_mapping_offset] + : 0u; + if (root.indirect_search_iterations == 0u || key_count < 2u || static_cast(root.indirect_mapping_offset) + 1u + - static_cast(root.indirect_mapping_capacity) * 2u > + static_cast(key_count) * 2u > next_snapshot.flattened_srt.size()) { return SpecializationFail("indirect image specialization has an invalid key mapping"); } - uint32_t exemplar = ImageResource::NoIndirectImage; - for (const auto resource: root.indirect_resources) { - if (resource >= next.images.size() || - next.images[resource].indirect_root != root_index) { - return SpecializationFail("indirect image specialization has an invalid candidate"); + uint32_t exemplar = ImageResource::NoIndirectImage; + uint32_t resource_count = 0; + for (uint32_t resource = 0; resource < next_specialization.images.size(); resource++) { + if (next_specialization.images[resource].indirect_root != root_index) { + continue; } - if (!NullImageDescriptor(next_snapshot.images[resource])) { + resource_count++; + if (exemplar == ImageResource::NoIndirectImage && + !NullImageDescriptor(next_snapshot.images[resource])) { exemplar = resource; - break; } } - if (exemplar == ImageResource::NoIndirectImage) { + if (resource_count < 2u || exemplar == ImageResource::NoIndirectImage) { return SpecializationFail("indirect image specialization has no typed candidate"); } - const auto& image_class = next.images[exemplar]; - for (uint32_t candidate = 0; candidate < next.images.size(); candidate++) { - auto& image = next.images[candidate]; + const auto& image_class = next_specialization.images[exemplar]; + for (uint32_t candidate = 0; candidate < next_specialization.images.size(); candidate++) { + auto& image = next_specialization.images[candidate]; if (image.indirect_root != root_index) { continue; } if (NullImageDescriptor(next_snapshot.images[candidate])) { - image.resource_class = image_class.resource_class; image.numeric_class = image_class.numeric_class; image.dimension = image_class.dimension; image.mip_count = image_class.mip_count; @@ -712,65 +547,225 @@ void SpecializeResources(Program& program, ResourceSnapshot& snapshot) { image.shader_swizzle = image_class.shader_swizzle; image.cube = image_class.cube; } - if (image.resource_class != image_class.resource_class || - image.numeric_class != image_class.numeric_class || + if (image.numeric_class != image_class.numeric_class || image.dimension != image_class.dimension || - image.mip_mode != image_class.mip_mode || image.mip_count != image_class.mip_count || image.conversion_format != image_class.conversion_format || image.shader_swizzle != image_class.shader_swizzle || - image.depth_compare != image_class.depth_compare || image.cube != image_class.cube) { return SpecializationFail( fmt::format("indirect image table at pc 0x{:08x} has incompatible candidates", - root.first_use_pc)); + program.info.images[root_index].first_use_pc)); } } } - auto memory_info = program.memory_info; + SamplerPlan sampler_plan; + if (!BuildSamplerPlan(program.info, next_specialization.images, sampler_plan)) { + return SpecializationFail("specialized sampler layout exceeds its resource limit"); + } + for (uint32_t index = 0; index < program.info.samplers.size(); index++) { + const auto target = sampler_plan.point_sampler[index]; + if (target != UINT32_MAX && target >= program.info.samplers.size()) { + next_snapshot.samplers.push_back(next_snapshot.samplers[index]); + } + } + specialization = std::move(next_specialization); + specialized_snapshot = std::move(next_snapshot); + return true; +} - struct SamplerUsage { - bool native = false; - bool point_only = false; - }; - std::vector sampler_usage(next.samplers.size()); - for (const auto& pair: next.sampled_pairs) { - if (pair.image >= next.images.size() || pair.sampler >= next.samplers.size()) { - return SpecializationFail( - fmt::format("sampled resource pair at pc 0x{:08x} is invalid", pair.first_use_pc)); +template +bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) { + if (base.samplers.size() > plan.point_sampler.size()) { + return false; + } + std::array usage {}; + plan.point_sampler.fill(UINT32_MAX); + plan.sampler_count = static_cast(base.samplers.size()); + for (const auto& pair: base.sampled_pairs) { + if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) { + return false; } - auto& usage = sampler_usage[pair.sampler]; - if (RequiresPointSampler(next.images[pair.image])) { - usage.point_only = true; + usage[pair.sampler] |= RequiresPointSampler(images[pair.image]) ? 2u : 1u; + } + for (uint32_t index = 0; index < base.samplers.size(); index++) { + if ((usage[index] & 2u) == 0u) { + continue; + } + if ((usage[index] & 1u) == 0u) { + plan.point_sampler[index] = index; } else { - usage.native = true; + if (plan.sampler_count >= ShaderInfo::MaxSamplers) { + return false; + } + plan.point_sampler[index] = plan.sampler_count++; } } - std::vector point_sampler(next.samplers.size(), UINT32_MAX); - for (uint32_t i = 0; i < sampler_usage.size(); i++) { - if (!sampler_usage[i].point_only) { + return true; +} + +ResourcePlan ExtractResourcePlan(const Program& program) { + ResourcePlan plan; + plan.stage = program.stage; + plan.shader_hash = program.shader_hash; + plan.user_data_base = program.user_data_base; + plan.user_data_count = program.user_data_count; + plan.info = program.info; + plan.memory_info = program.memory_info; + plan.srt_plan_complete = program.srt_plan_complete; + plan.resource_tracking_complete = program.resource_tracking_complete; + + std::unordered_map cloned; + std::function Clone = [&](Value value) -> Value { + value = value.Resolve(); + const auto* source = value.TryInstruction(); + if (source == nullptr) { + return value; + } + if (source->GetOpcode() == ValueOpcode::Phi) { + const auto invariant = ResolveInvariantPhi(program, value); + if (!invariant.IsEmpty() && invariant != value) { + return Clone(invariant); + } + } + if (const auto found = cloned.find(source); found != cloned.end()) { + return Value(found->second); + } + auto& target = + plan.value_storage.emplace_back(source->GetOpcode(), source->Flags()); + cloned.emplace(source, &target); + if (source->GetOpcode() == ValueOpcode::Phi) { + for (size_t index = 0; index < source->NumArgs(); index++) { + target.AddPhiOperand(nullptr, Clone(source->Arg(index))); + } + } else { + for (size_t index = 0; index < source->NumArgs(); index++) { + target.SetArg(index, Clone(source->Arg(index))); + } + } + return Value(&target); + }; + + plan.descriptor_sources.reserve(program.descriptor_sources.size()); + for (const auto& source: program.descriptor_sources) { + auto& target = plan.descriptor_sources.emplace_back(); + target.dword_count = source.dword_count; + target.indirect_image = source.indirect_image; + for (uint32_t dword = 0; dword < source.dword_count; dword++) { + target.dwords[dword] = Clone(source.dwords[dword]); + } + } + plan.srt_reads.reserve(program.srt_reads.size()); + for (const auto& read: program.srt_reads) { + plan.srt_reads.push_back({Clone(read.value), read.flat_offset}); + } + plan.materialization_sources.reserve(plan.info.buffers.size() + plan.info.images.size() + + plan.info.samplers.size()); + for (const auto& buffer: plan.info.buffers) { + plan.materialization_sources.push_back(buffer.source); + } + for (const auto& image: plan.info.images) { + const auto* source = Source(plan, image.source); + if (source != nullptr && source->indirect_image.has_value()) { + plan.requires_specialization_memory = true; + } else { + plan.materialization_sources.push_back(image.source); + } + } + for (const auto& sampler: plan.info.samplers) { + plan.materialization_sources.push_back(sampler.source); + } + plan.clean_flat_slots.resize(plan.srt_reads.size()); + for (const auto& image: plan.info.images) { + const auto* source = Source(plan, image.source); + if (source == nullptr || !source->indirect_image.has_value()) { continue; } - if (!sampler_usage[i].native) { - next.samplers[i].force_point_filtering = true; - point_sampler[i] = i; + MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source), + plan.clean_flat_slots); + MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->heap_source), + plan.clean_flat_slots); + } + return plan; +} + +bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, + ResourceSnapshot& snapshot, ResourceSpecialization& specialization) { + MaterializedSnapshot materialized; + if (!MaterializeSnapshot(program, runtime, materialized)) { + return false; + } + return BuildResourceSpecialization(program, std::move(materialized), snapshot, specialization); +} + +void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) { + EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete || + program.binding_layout_complete); + EXIT_IF(program.info.buffers.size() != specialization.buffers.size() || + program.info.images.size() > specialization.images.size()); + + auto buffers = program.info.buffers; + for (size_t index = 0; index < buffers.size(); index++) { + buffers[index].packed_stride = specialization.buffers[index].packed_stride; + buffers[index].descriptor_format = specialization.buffers[index].descriptor_format; + buffers[index].descriptor_swizzle = specialization.buffers[index].descriptor_swizzle; + } + auto images = program.info.images; + images.reserve(specialization.images.size()); + for (uint32_t index = 0; index < specialization.images.size(); index++) { + const auto& source = specialization.images[index]; + if (index >= images.size()) { + EXIT_IF(source.indirect_root >= program.info.images.size()); + images.push_back(program.info.images[source.indirect_root]); + } + auto& image = images[index]; + image.numeric_class = source.numeric_class; + image.dimension = source.dimension; + image.mip_count = source.mip_count; + image.conversion_format = source.conversion_format; + image.shader_swizzle = source.shader_swizzle; + image.indirect_root = source.indirect_root; + image.indirect_mapping_offset = source.indirect_mapping_offset; + image.indirect_search_iterations = source.indirect_search_iterations; + image.cube = source.cube; + image.indirect_resources.clear(); + } + for (uint32_t index = 0; index < images.size(); index++) { + const auto root = images[index].indirect_root; + if (root != ImageResource::NoIndirectImage) { + EXIT_IF(root >= images.size()); + images[root].indirect_resources.push_back(index); + } + } + + SamplerPlan sampler_plan; + EXIT_IF(!BuildSamplerPlan(program.info, images, sampler_plan)); + auto samplers = program.info.samplers; + auto sampled_pairs = program.info.sampled_pairs; + samplers.reserve(sampler_plan.sampler_count); + for (uint32_t index = 0; index < program.info.samplers.size(); index++) { + const auto target = sampler_plan.point_sampler[index]; + if (target == UINT32_MAX) { continue; } - if (next.samplers.size() >= ShaderInfo::MaxSamplers) { - return SpecializationFail( - "point-filter sampler variants exceed the dense sampler resource limit"); - } - point_sampler[i] = static_cast(next.samplers.size()); - auto sampler = next.samplers[i]; - sampler.force_point_filtering = true; - next.samplers.push_back(sampler); - next_snapshot.samplers.push_back(next_snapshot.samplers[i]); - } - for (auto& pair: next.sampled_pairs) { - if (RequiresPointSampler(next.images[pair.image])) { - pair.sampler = point_sampler[pair.sampler]; + if (target == index) { + samplers[index].force_point_filtering = true; + } else { + EXIT_IF(target != samplers.size()); + auto sampler = samplers[index]; + sampler.force_point_filtering = true; + samplers.push_back(std::move(sampler)); } } + for (auto& pair: sampled_pairs) { + if (RequiresPointSampler(images[pair.image])) { + EXIT_IF(sampler_plan.point_sampler[pair.sampler] == UINT32_MAX); + pair.sampler = sampler_plan.point_sampler[pair.sampler]; + } + samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare; + } + + auto memory_info = program.memory_info; for (const auto* block: program.blocks) { for (const auto& inst: *block) { const auto image_opcode = ImageOpcodeInfoOf(inst.GetOpcode()); @@ -778,31 +773,24 @@ void SpecializeResources(Program& program, ResourceSnapshot& snapshot) { continue; } const auto index = inst.Flags().index; - if (index >= memory_info.size()) { - return SpecializationFail( - fmt::format("typed image instruction has invalid memory metadata {}", index)); - } + EXIT_IF(index >= memory_info.size()); auto& memory = memory_info[index]; - if (memory.resource >= next.images.size()) { - return SpecializationFail(fmt::format( - "typed image instruction has invalid resource {}", memory.resource)); - } - const auto& image = next.images[memory.resource]; + EXIT_IF(memory.resource >= images.size()); + const auto& image = images[memory.resource]; if (image_opcode.needs_sampler && RequiresPointSampler(image) && - memory.sampler < point_sampler.size()) { - memory.sampler = point_sampler[memory.sampler]; - } - if (image.indirect_root == memory.resource && - inst.GetOpcode() != ValueOpcode::ImageSampleRaw) { - return SpecializationFail( - fmt::format("indirect image table at pc 0x{:08x} is used by unsupported {}", - inst.Flags().pc, ValueOpcodeName(inst.GetOpcode()))); + memory.sampler < program.info.samplers.size()) { + EXIT_IF(sampler_plan.point_sampler[memory.sampler] == UINT32_MAX); + memory.sampler = sampler_plan.point_sampler[memory.sampler]; } + EXIT_IF(image.indirect_root == memory.resource && + inst.GetOpcode() != ValueOpcode::ImageSampleRaw); } } - program.info = std::move(next); - program.memory_info = std::move(memory_info); - snapshot = std::move(next_snapshot); + program.info.buffers = std::move(buffers); + program.info.images = std::move(images); + program.info.samplers = std::move(samplers); + program.info.sampled_pairs = std::move(sampled_pairs); + program.memory_info = std::move(memory_info); } } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h index 9f61cf4d8..5aa3d55c5 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.h @@ -5,34 +5,46 @@ namespace Libs::Graphics::ShaderRecompiler::IR { -struct ResourceSnapshot { - struct IndirectImage { - uint32_t resource = 0; - uint32_t capacity = 0; - std::vector keys; - std::vector candidates; - std::vector descriptors; +// Canonical module-affecting resource state. Runtime addresses and descriptor payloads remain in +// ResourceSnapshot and therefore do not create shader permutations. +struct ResourceSpecialization { + struct Buffer { + uint32_t packed_stride = 0; + Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid; + uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7); + bool operator==(const Buffer&) const = default; }; - std::vector buffers; - std::vector images; - std::vector samplers; - std::vector flattened_srt; - std::vector user_data; - std::vector indirect_images; + struct Image { + Prospero::TextureNumericClass numeric_class = Prospero::TextureNumericClass::Unsupported; + Decoder::ImageDimension dimension = Decoder::ImageDimension::Unknown; + uint32_t mip_count = 1; + Prospero::BufferFormat conversion_format = Prospero::BufferFormat::kInvalid; + uint32_t shader_swizzle = ShaderImageIdentitySwizzle; + uint32_t indirect_root = ImageResource::NoIndirectImage; + uint32_t indirect_mapping_offset = 0; + uint32_t indirect_search_iterations = 0; + bool cube = false; + bool operator==(const Image&) const = default; + }; + + std::vector buffers; + std::vector images; + + bool operator==(const ResourceSpecialization&) const = default; }; -bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot); -bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot); +// Extracts the descriptor/SRT value graph before resource specialization. The returned plan owns +// its values and is independent of the translated shader CFG. +ResourcePlan ExtractResourcePlan(const Program& program); -// Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot. -// On failure the destination is unchanged. -bool MaterializeResources(const Program& program, const SrtRuntime& runtime, - ResourceSnapshot& snapshot); +// Resolves and specializes the immutable resource plan in one transaction. On failure both +// destinations are unchanged. +bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime, + ResourceSnapshot& snapshot, ResourceSpecialization& specialization); -// Applies runtime descriptor shape/format facts to a copied dense topology before layout and -// emission. Invalid specializations are fatal. -void SpecializeResources(Program& program, ResourceSnapshot& snapshot); +// Applies an already-derived specialization to native IR before layout and emission. +void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp index e70462b81..f97942725 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp @@ -464,6 +464,16 @@ private: DescriptorSource descriptor; MakeSource(*handle, width, sampler, sample_adjust, descriptor, pc); uint32_t bad_dword = 0; + if (expected == ValueOpcode::GetImageResource) { + for (; bad_dword < descriptor.dword_count; bad_dword++) { + const auto* value = descriptor.dwords[bad_dword].Resolve().TryInstruction(); + if (value != nullptr && value->GetOpcode() == ValueOpcode::ReadConstBuffer) { + Fail(pc, fmt::format("{} dword {} is not a valid runtime value", + ValueOpcodeName(expected), bad_dword)); + } + } + bad_dword = 0; + } if (!ValidateSource(descriptor, bad_dword)) { Fail(pc, fmt::format("{} dword {} is not a valid runtime value", ValueOpcodeName(expected), bad_dword)); diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp index 8094e2381..47f2b5a25 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.cpp @@ -26,7 +26,7 @@ const char* StageName(ShaderType stage) { } } -std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) { +std::string Diagnostic(const ResourcePlan& program, uint32_t pc, const std::string& message) { return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash, StageName(program.stage), pc, message); } @@ -51,7 +51,7 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) { return true; } -bool IsRawRead(const Program& values, const Inst& inst) { +bool IsRawRead(const ResourcePlan& values, const Inst& inst) { const auto op = inst.GetOpcode(); if (op != ValueOpcode::LoadAddressU32 && op != ValueOpcode::ReadConstBuffer) { return false; @@ -134,7 +134,7 @@ bool IsRuntimeUniformOp(ValueOpcode op) { class RuntimeValidator { public: - explicit RuntimeValidator(const Program& program, Value active_mask = {}) + explicit RuntimeValidator(const ResourcePlan& program, Value active_mask = {}) : m_program(program), m_active_mask(active_mask.Resolve()) {} bool Run(Value value) { return Validate(value); } @@ -262,7 +262,7 @@ private: return finish(true); } - const Program& m_program; + const ResourcePlan& m_program; Value m_active_mask; std::unordered_set m_visiting; }; @@ -415,7 +415,7 @@ private: class Evaluator { public: - Evaluator(const Program& program, const SrtRuntime& runtime, + Evaluator(const ResourcePlan& program, const SrtRuntime& runtime, std::span clean_flat_slots = {}, Evaluator* clean_evaluator = nullptr, Value active_mask = {}) : m_program(program), m_runtime(runtime), m_clean_flat_slots(clean_flat_slots), @@ -435,9 +435,7 @@ private: return std::bit_cast(static_cast(bits)); } - static uint64_t Float32Bits(float value) { - return std::bit_cast(value); - } + static uint64_t Float32Bits(float value) { return std::bit_cast(value); } bool EvaluateWide(Value value, uint64_t& result) { value = value.Resolve(); @@ -456,6 +454,11 @@ private: if (inst == nullptr) { return false; } + if (!m_reserved) { + m_cache.reserve(m_program.value_storage.size()); + m_visiting.reserve(m_program.value_storage.size()); + m_reserved = true; + } if (!m_active_mask.IsEmpty() && IsRuntimeSelect(inst->GetOpcode()) && inst->NumArgs() == 3 && inst->Arg(0).Resolve() == m_active_mask) { return EvaluateWide(inst->Arg(1), result); @@ -915,24 +918,24 @@ private: return false; } - const Program& m_program; + const ResourcePlan& m_program; const SrtRuntime& m_runtime; std::span m_clean_flat_slots; Evaluator* m_clean_evaluator = nullptr; Value m_active_mask; std::unordered_map m_cache; std::vector m_visiting; + bool m_reserved = false; }; -const DescriptorSource* Source(const Program& program, uint32_t source) { +const DescriptorSource* Source(const ResourcePlan& program, uint32_t source) { if (source >= program.descriptor_sources.size()) { return nullptr; } return &program.descriptor_sources[source]; } -bool EvaluateRuntimeSourcesImpl(const Program& program, - std::span requests, +bool EvaluateRuntimeSourcesImpl(const ResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluate_flat, std::span clean_flat_slots) { @@ -948,9 +951,9 @@ bool EvaluateRuntimeSourcesImpl(const Program& program Evaluator clean_evaluator(program, clean_runtime); Evaluator evaluator(program, runtime, clean_flat_slots, &clean_evaluator); std::vector evaluated; - evaluated.reserve(requests.size()); - for (const auto& request: requests) { - const auto* source = Source(program, request.source); + evaluated.reserve(sources.size()); + for (const auto source_index: sources) { + const auto* source = Source(program, source_index); if (source == nullptr) { return false; } @@ -985,7 +988,7 @@ bool EvaluateRuntimeSourcesImpl(const Program& program } // namespace -bool ValidateRuntimeValue(const Program& program, Value value) { +bool ValidateRuntimeValue(const ResourcePlan& program, Value value) { return RuntimeValidator(program).Run(value); } @@ -998,34 +1001,32 @@ void BuildSrtPlan(Program& program) { program.srt_plan_complete = true; } -bool EvaluateDescriptorSource(const Program& program, uint32_t source, const SrtRuntime& runtime, - DescriptorValue& result) { - const DescriptorSourceRequest request {source}; - std::vector results; - if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results)) { +bool EvaluateDescriptorSource(const ResourcePlan& program, uint32_t source, + const SrtRuntime& runtime, DescriptorValue& result) { + std::vector results; + if (!EvaluateDescriptorSources(program, std::span {&source, 1}, runtime, results)) { return false; } result = results.front(); return true; } -bool EvaluateDescriptorSources(const Program& program, - std::span requests, +bool EvaluateDescriptorSources(const ResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results) { std::vector ignored; - return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, {}); + return EvaluateRuntimeSourcesImpl(program, sources, runtime, results, ignored, false, {}); } -bool EvaluateRuntimeSources(const Program& program, - std::span requests, +bool EvaluateRuntimeSources(const ResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span clean_flat_slots) { - return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, + return EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, clean_flat_slots); } -bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector& flat) { +bool WalkSrt(const ResourcePlan& program, const SrtRuntime& runtime, + std::vector& flat) { std::vector ignored; return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, {}); } diff --git a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h index 9ef430e3c..839a69fb3 100644 --- a/src/graphics/shader/recompiler/ir/passes/SrtWalker.h +++ b/src/graphics/shader/recompiler/ir/passes/SrtWalker.h @@ -19,32 +19,27 @@ struct SrtRuntime { SrtMemoryReader read_specialization_memory = nullptr; }; -struct DescriptorSourceRequest { - uint32_t source = 0; -}; - // Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots; // dynamic offsets remain explicit and are never assigned a fake slot. void BuildSrtPlan(Program& program); -bool ValidateRuntimeValue(const Program& program, Value value); +bool ValidateRuntimeValue(const ResourcePlan& program, Value value); -bool EvaluateDescriptorSource(const Program& program, uint32_t source, const SrtRuntime& runtime, - DescriptorValue& result); +bool EvaluateDescriptorSource(const ResourcePlan& program, uint32_t source, + const SrtRuntime& runtime, DescriptorValue& result); // Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by // several descriptors are memoized once across the batch. -bool EvaluateDescriptorSources(const Program& program, - std::span requests, +bool EvaluateDescriptorSources(const ResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results); // Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk. // On failure neither destination is changed. -bool EvaluateRuntimeSources(const Program& program, - std::span requests, +bool EvaluateRuntimeSources(const ResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span clean_flat_slots); -bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector& flat); +bool WalkSrt(const ResourcePlan& program, const SrtRuntime& runtime, + std::vector& flat); } // namespace Libs::Graphics::ShaderRecompiler::IR diff --git a/src/graphics/shader/rectListShader.cpp b/src/graphics/shader/rectListShader.cpp index ef791886e..81be9a0bb 100644 --- a/src/graphics/shader/rectListShader.cpp +++ b/src/graphics/shader/rectListShader.cpp @@ -48,7 +48,7 @@ std::vector GetParameters(const ShaderVertexInputInfo& vertex_info, std::vector parameters; for (const auto input: active_inputs) { const auto input_location = ShaderPixelParameterMappedLocation(*pixel_info, input); - if ((vertex_info.param_export_mask & (1u << input_location)) != 0) { + if ((vertex_info.stage.program->param_export_mask & (1u << input_location)) != 0) { parameters.push_back({input_location, ShaderPixelParameterLocation(*pixel_info, active_inputs, input), ShaderPixelParameterIsFlat(*pixel_info, input)}); diff --git a/src/graphics/shader/shader.cpp b/src/graphics/shader/shader.cpp index 8bb5a1bfa..0eed19cbd 100644 --- a/src/graphics/shader/shader.cpp +++ b/src/graphics/shader/shader.cpp @@ -3,7 +3,6 @@ #include "common/assert.h" #include "common/common.h" #include "common/emulatorConfig.h" -#include "common/file.h" #include "common/logging/log.h" #include "common/magicEnum.h" #include "common/profiler.h" @@ -12,23 +11,15 @@ #include "graphics/guest_gpu/graphicsRun.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/host_gpu/renderer/renderContext.h" -#include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h" -#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/shaderCompiler.h" #include "graphics/shader/shaderVertexMetadata.h" -#include "kernel/memory.h" #include "libs/errno.h" -#include "spirv-tools/libspirv.h" -#include "spirv-tools/libspirv.hpp" -#include "spirv-tools/optimizer.hpp" #include #include #include #include -#include -#include #include #include #include @@ -49,15 +40,6 @@ namespace Libs::Graphics { -namespace { - -bool ReadShaderGuestMemory(void*, uint64_t address, uint32_t* value) { - return value != nullptr && - Libs::LibKernel::Memory::TryReadGpuCleanBacking(address, value, sizeof(*value)); -} - -} // namespace - struct ShaderBinaryInfo { uint8_t signature[7]; uint8_t version; @@ -107,62 +89,6 @@ static ShaderMappedData ShaderGetMappedData(uint64_t addr, const char* label) { EXIT("%s shader=0x%016" PRIx64 " is missing from ShaderMap\n", label, addr); } -static bool SpirvDisassemble(const uint32_t* src_binary, size_t src_binary_size, - std::string* dst_disassembly) { - if (dst_disassembly != nullptr) { - spvtools::SpirvTools core(SPV_ENV_VULKAN_1_2); - - std::string disassembly; - if (!core.Disassemble(src_binary, src_binary_size, &disassembly, - static_cast(SPV_BINARY_TO_TEXT_OPTION_NO_HEADER) | - static_cast(SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES) | - static_cast(SPV_BINARY_TO_TEXT_OPTION_COMMENT) | - static_cast(SPV_BINARY_TO_TEXT_OPTION_INDENT) | - static_cast(SPV_BINARY_TO_TEXT_OPTION_COLOR))) { - *dst_disassembly = disassembly.c_str(); - - LOGF("Disassemble failed\n"); - return false; - } - - *dst_disassembly = disassembly.c_str(); - } - return true; -} - -static bool SpirvValidateBinary(const char* label, uint64_t shader_hash, - const std::vector& spirv) { - if (!Config::ShaderValidationEnabled()) { - return true; - } - - spvtools::SpirvTools core(SPV_ENV_VULKAN_1_3); - std::string messages; - core.SetMessageConsumer([&messages](spv_message_level_t /*level*/, const char* /*source*/, - const spv_position_t& position, const char* message) { - messages += fmt::format("{}: {} ({}) {}\n", static_cast(position.line), - static_cast(position.column), static_cast(position.index), - message); - }); - - if (core.Validate(spirv)) { - return true; - } - - std::string disassembly; - SpirvDisassemble(spirv.data(), spirv.size(), &disassembly); - LOGF_COLOR(Log::Color::BrightRed, "%s SPIR-V validation failed hash=0x%016" PRIx64 ":\n%s", - label, shader_hash, messages.c_str()); - LOGF("%s\n", disassembly.c_str()); - return false; -} - -static void ExitShaderRecompilerFailure(const char* label, uint64_t shader_hash, - const char* reason) { - EXIT("%s failed hash=0x%016" PRIx64 ": %s\n", label, shader_hash, - reason != nullptr ? reason : ""); -} - static const ShaderBinaryInfo* GetBinaryInfo(const uint32_t* code) { EXIT_IF(code == nullptr); @@ -695,7 +621,6 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo& regs, info = {}; - info.export_count = static_cast(sh.GetExportCount()); info.pa_cl_vs_out_cntl = sh.m_paClVsOutCntl; EXIT_NOT_IMPLEMENTED(regs.es_regs.data_addr == 0); @@ -746,18 +671,12 @@ static bool ShaderGetStaticInputInfoVS(const HW::VertexShaderInfo& regs, static void ShaderGetStaticInputInfoPS( const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vs_info, std::span target_export_mapping, const ShaderMappedData& data, ShaderPixelInputInfo& ps_info) { KYTY_PROFILER_FUNCTION(); ps_info = {}; ps_info.scratch_size_dwords = data.scratch_size_dwords; - ps_info.push_constant_offset = - vs_info.stage.program != nullptr - ? vs_info.stage.program->bindings.push_constant_offset + - vs_info.stage.program->bindings.push_constant_size - : 0; // SPI_PS_IN_CONTROL.NUM_INTERP occupies bits 5:0. Keep the remaining control // flags in the hardware state and extract only the input count here. @@ -801,7 +720,6 @@ static void ShaderGetStaticInputInfoPS( ? target_export_mapping[i] : Prospero::ColorComponentMapping {}; } - ps_info.mrt_output_mask = 0; } static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, @@ -828,73 +746,12 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs, info.workgroup_register = regs.cs_regs.user_sgpr; } -static void ApplyVertexOutputs(ShaderVertexInputInfo& info, - const ShaderRecompiler::IR::Program& program) { - info.export_count = 0; - info.param_export_mask = 0; - for (const auto& output: program.info.outputs) { - if (output.kind == ShaderRecompiler::IR::StageOutputKind::Parameter && output.index < 32) { - info.param_export_mask |= 1u << output.index; - info.export_count = std::max(info.export_count, static_cast(output.index + 1)); - } - } -} - -static void ApplyPixelOutputs(ShaderPixelInputInfo& info, - const ShaderRecompiler::IR::Program& program) { - info.mrt_output_mask = 0; - for (const auto& output: program.info.outputs) { - if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt && output.index < 8) { - info.mrt_output_mask |= 1u << output.index; - } - } -} - -static bool LogPermutationMismatch(const ShaderRecompiler::IR::Program* program, - const char* stage) { - static std::atomic log_count {0}; - if (Config::GraphicsDebugDumpEnabled() && - log_count.fetch_add(1, std::memory_order_relaxed) < 64) { - LOGF("ShaderProgramCache native runtime mismatch %s shader=0x%016" PRIx64 "\n", - stage, program != nullptr ? program->shader_hash : 0); - } - return false; -} - -bool MaterializeProgram(const std::shared_ptr& program, - const ShaderParams& params, ShaderVertexInputInfo& info) { - if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, - ReadShaderGuestMemory)) { - return LogPermutationMismatch(program.get(), "VS"); - } - ApplyVertexOutputs(info, *program); - return true; -} - -bool MaterializeProgram(const std::shared_ptr& program, - const ShaderParams& params, ShaderPixelInputInfo& info) { - if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, - ReadShaderGuestMemory)) { - return LogPermutationMismatch(program.get(), "PS"); - } - ApplyPixelOutputs(info, *program); - return true; -} - -bool MaterializeProgram(const std::shared_ptr& program, - const ShaderParams& params, ShaderComputeInputInfo& info) { - if (!ShaderMaterializeStageRuntime(program, params.user_data, params.Base(), info.stage, - ReadShaderGuestMemory)) { - return LogPermutationMismatch(program.get(), "CS"); - } - return true; -} - void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector& key) { - EXIT_IF(info.resources_num < 0 || info.resources_num > ShaderVertexInputInfo::RES_MAX || - info.buffers_num < 0 || info.buffers_num > ShaderVertexInputInfo::RES_MAX); + EXIT_IF(info.resources_num < 0 || info.resources_num > ShaderVertexInputInfo::RES_MAX); key.clear(); key.push_back(static_cast(info.fetch_embedded)); + key.push_back(static_cast(info.fetch_attrib_reg)); + key.push_back(static_cast(info.fetch_buffer_reg)); key.push_back(info.resources_num); key.push_back(info.scratch_size_dwords); key.push_back(info.pa_cl_vs_out_cntl); @@ -917,6 +774,7 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector(destination.attr_id)); key.push_back(resource.Stride()); key.push_back(static_cast(resource.SwizzleEnabled())); key.push_back(resource.DstSelX()); @@ -927,28 +785,11 @@ void BuildStageStaticKey(const ShaderVertexInputInfo& info, std::vector(resource.AddTid())); } - - key.push_back(info.buffers_num); - int attributes_num = 0; - for (int i = 0; i < info.buffers_num; i++) { - const auto& buffer = info.buffers[i]; - EXIT_IF(buffer.attr_num < 0 || buffer.attr_num > ShaderVertexInputBuffer::ATTR_MAX); - attributes_num += buffer.attr_num; - EXIT_IF(attributes_num > ShaderVertexInputInfo::RES_MAX); - key.push_back(buffer.attr_num); - key.push_back(buffer.stride); - key.push_back(buffer.fetch_index); - for (int j = 0; j < buffer.attr_num; j++) { - key.push_back(buffer.attr_indices[j]); - key.push_back(buffer.attr_offsets[j]); - } - } } void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector& key) { EXIT_IF(info.input_num > std::size(info.interpolator_settings)); key.clear(); - key.push_back(info.push_constant_offset); key.push_back(info.scratch_size_dwords); key.push_back(info.input_num); key.push_back(info.ps_system_input_base); @@ -1006,11 +847,10 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRe ShaderParams PrepareProgram( const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vs_info, std::span target_export_mapping, ShaderPixelInputInfo& ps_info) { const auto data = ShaderGetMappedData(regs.ps_regs.data_addr, "ShaderGetInputInfoPS():"); - ShaderGetStaticInputInfoPS(regs, sh, vs_info, target_export_mapping, data, ps_info); + ShaderGetStaticInputInfoPS(regs, sh, target_export_mapping, data, ps_info); return GetShaderParams( regs.ps_regs.data_addr, "ShaderRecompiler PS", GetDeclaredShaderHash(regs.ps_regs.data_addr), std::span(regs.ps_user_sgpr.value, regs.ps_regs.rsrc2.user_sgpr), data); @@ -1032,11 +872,8 @@ void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo& info) { LOGF("ShaderDbgDumpInputInfo()\n"); LOGF("\t fetch_external = %s\n" - "\t fetch_embedded = %s\n" - "\t param_export_mask = 0x%08" PRIx32 "\n" - "\t export_count = %d\n", - info.fetch_external ? "true" : "false", info.fetch_embedded ? "true" : "false", - info.param_export_mask, info.export_count); + "\t fetch_embedded = %s\n", + info.fetch_external ? "true" : "false", info.fetch_embedded ? "true" : "false"); for (int i = 0; i < info.resources_num; i++) { LOGF("\t input %d\n", i); @@ -1134,205 +971,6 @@ void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info) { info.group_id[1] ? "true" : "false", info.group_id[2] ? "true" : "false"); } -static bool ShaderRecompilerTextDumpEnabled() { - // Graphics debug dump already writes SPIR-V binaries to _Shaders. Keep the very large - // disassembly/IR text behind the shader-log switch so file logging cannot stall boot. - return Config::GetShaderLogDirection() != Config::ShaderLogDirection::Silent; -} - -static void DumpShaderRecompilerSpirv(const char* type, uint64_t shader_hash, - const std::vector& bin) { - if (!Config::GraphicsDebugDumpEnabled()) { - return; - } - - static std::atomic_int id = 0; - - const auto base_name = Config::GetShaderLogFolder() / - fmt::format("{:04d}_new_shader_{}_{:016x}", id++, type, shader_hash); - Common::File::CreateDirectories(base_name.parent_path()); - - Common::File spv_file; - auto spv_name = base_name; - spv_name += ".spv"; - spv_file.Create(spv_name); - if (spv_file.IsInvalid()) { - auto spv_name_text = Common::PathToString(spv_name); - LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", spv_name_text.c_str()); - } else { - spv_file.Write(bin.data(), bin.size() * 4); - spv_file.Close(); - } - - return; - - std::string text; - if (!SpirvDisassemble(bin.data(), bin.size(), &text)) { - auto spv_name_text = Common::PathToString(spv_name); - LOGF_COLOR(Log::Color::BrightRed, "SpirvDisassemble() failed for %s\n", - spv_name_text.c_str()); - return; - } - - Common::File asm_file; - auto asm_name = base_name; - asm_name += ".spvasm"; - asm_file.Create(asm_name); - if (asm_file.IsInvalid()) { - auto asm_name_text = Common::PathToString(asm_name); - LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", asm_name_text.c_str()); - } else { - asm_file.Printf("%s", text.c_str()); - asm_file.Close(); - } -} - -static void DumpShaderRecompilerOriginal(const char* type, uint64_t shader_hash, - std::span code, - const std::string& decoded_dump) { - if (!Config::GraphicsDebugDumpEnabled()) { - return; - } - EXIT_IF(code.empty()); - - static std::atomic_int id = 0; - - const auto base_name = Config::GetShaderLogFolder() / "original" / - fmt::format("{:04d}_new_shader_{}_{:016x}", id++, type, shader_hash); - Common::File::CreateDirectories(base_name.parent_path()); - - Common::File bin_file; - auto bin_name = base_name; - bin_name += ".bin"; - bin_file.Create(bin_name); - if (bin_file.IsInvalid()) { - auto bin_name_text = Common::PathToString(bin_name); - LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", bin_name_text.c_str()); - } else { - bin_file.Write(code.data(), code.size_bytes()); - bin_file.Close(); - } - if (decoded_dump.empty()) { - return; - } - - Common::File text_file; - auto text_name = base_name; - text_name += ".rdna2"; - text_file.Create(text_name); - if (text_file.IsInvalid()) { - auto text_name_text = Common::PathToString(text_name); - LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", text_name_text.c_str()); - } else { - text_file.Printf("%s", decoded_dump.c_str()); - text_file.Close(); - } -} - -static const char* ShaderStageName(ShaderType stage) { - switch (stage) { - case ShaderType::Vertex: return "vs"; - case ShaderType::Pixel: return "ps"; - case ShaderType::Compute: return "cs"; - default: EXIT("invalid shader stage\n"); - } -} - -static const char* ShaderStageLabel(ShaderType stage) { - switch (stage) { - case ShaderType::Vertex: return "ShaderRecompiler VS"; - case ShaderType::Pixel: return "ShaderRecompiler PS"; - case ShaderType::Compute: return "ShaderRecompiler CS"; - default: EXIT("invalid shader stage\n"); - } -} - -static ShaderRecompiler::CompileOptions MakeCompileOptions(const ShaderParams& params, - ShaderType stage) { - ShaderRecompiler::CompileOptions options; - options.stage = stage; - options.shader_hash = params.hash; - options.user_data = params.user_data; - options.read_specialization_memory = ReadShaderGuestMemory; - options.dump_ir = ShaderRecompilerTextDumpEnabled(); - options.early_dump = options.dump_ir; - options.dump_label = ShaderStageLabel(stage); - return options; -} - -static vk::ShaderModule CompileModule(vk::Device device, const ShaderParams& params, - ShaderRecompiler::CompileOptions options, - ShaderStageRuntime& stage) { - const auto* stage_name = ShaderStageName(options.stage); - const auto* label = ShaderStageLabel(options.stage); - auto result = ShaderRecompiler::Recompile(params.code, options); - - DumpShaderRecompilerOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump); - if (!SpirvValidateBinary(label, options.shader_hash, result.spirv)) { - DumpShaderRecompilerSpirv(stage_name, options.shader_hash, result.spirv); - ExitShaderRecompilerFailure(label, options.shader_hash, "SPIR-V validation failed"); - } - DumpShaderRecompilerSpirv(stage_name, options.shader_hash, result.spirv); - - stage.program = - std::make_shared(std::move(result.program)); - stage.resources = - std::make_shared(std::move(result.resources)); - - vk::ShaderModuleCreateInfo create_info {}; - create_info.sType = vk::StructureType::eShaderModuleCreateInfo; - create_info.codeSize = result.spirv.size() * sizeof(uint32_t); - create_info.pCode = result.spirv.data(); - vk::ShaderModule module = nullptr; - RequireVulkanSuccess(device.createShaderModule(&create_info, nullptr, &module), - "create recompiled shader module"); - EXIT_IF(module == nullptr); - - if (options.dump_ir) { - if (!options.early_dump) { - LOGF("%s decoded RDNA2:\n%s", label, result.decoded_dump.c_str()); - LOGF("%s IR:\n%s", label, result.ir_dump.c_str()); - } - LOGF("%s SPIR-V words=%" PRIu64 " wave_size=%u\n", label, - static_cast(result.spirv.size()), options.wave_size); - } - return module; -} - -vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, - ShaderVertexInputInfo& input_info) { - KYTY_PROFILER_FUNCTION(profiler::colors::Amber300); - auto options = MakeCompileOptions(params, ShaderType::Vertex); - options.user_data_base = 8; - options.scratch_dwords = input_info.scratch_size_dwords; - options.input_info.vertex = &input_info; - const auto module = CompileModule(device, params, options, input_info.stage); - ApplyVertexOutputs(input_info, *input_info.stage.program); - return module; -} - -vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, - ShaderPixelInputInfo& input_info) { - KYTY_PROFILER_FUNCTION(profiler::colors::Blue300); - auto options = MakeCompileOptions(params, ShaderType::Pixel); - options.scratch_dwords = input_info.scratch_size_dwords; - options.push_constant_offset = input_info.push_constant_offset; - options.input_info.pixel = &input_info; - const auto module = CompileModule(device, params, options, input_info.stage); - ApplyPixelOutputs(input_info, *input_info.stage.program); - return module; -} - -vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, - ShaderComputeInputInfo& input_info) { - KYTY_PROFILER_FUNCTION(profiler::colors::CyanA700); - auto options = MakeCompileOptions(params, ShaderType::Compute); - options.scratch_dwords = input_info.scratch_size_dwords; - options.wave_size = input_info.wave_size; - options.input_info.compute = &input_info; - return CompileModule(device, params, options, input_info.stage); -} - bool ShaderAddressValid(uint64_t addr) { return reinterpret_cast(addr) != nullptr; } diff --git a/src/graphics/shader/shader.h b/src/graphics/shader/shader.h index 7790724ef..4e51d211f 100644 --- a/src/graphics/shader/shader.h +++ b/src/graphics/shader/shader.h @@ -4,10 +4,10 @@ #include "common/abi.h" #include "common/common.h" #include "graphics/guest_gpu/gpu_defs.h" +#include "graphics/shader/recompiler/ir/ResourceSnapshot.h" #include "graphics/shader/shaderBindings.h" #include -#include #include #include #include @@ -36,30 +36,18 @@ struct ShaderRegisters; enum class ShaderType { Unknown, Vertex, Pixel, Fetch, Compute }; namespace ShaderRecompiler::IR { -struct Program; -struct ResourceSnapshot; +struct CompiledShaderInfo; } // namespace ShaderRecompiler::IR struct ShaderStageRuntime { - std::shared_ptr program; - std::shared_ptr resources; + const ShaderRecompiler::IR::CompiledShaderInfo* program = nullptr; + ShaderRecompiler::IR::ResourceSnapshot resources; [[nodiscard]] explicit operator bool() const { - return program != nullptr && resources != nullptr; + return program != nullptr; } }; -using ShaderSpecializationMemoryReader = bool (*)(void* userdata, uint64_t address, - uint32_t* value); - -// Resolves an immutable native shader plan against current user data. The prior stage is preserved -// if any ReadConst, snapshot, or specialization check fails. -bool ShaderMaterializeStageRuntime( - std::shared_ptr program, - std::span user_data, uint64_t shader_base, ShaderStageRuntime& stage, - ShaderSpecializationMemoryReader read_specialization_memory = nullptr, - void* read_memory_data = nullptr); - constexpr uint32_t DstSel(uint32_t x, uint32_t y = 0, uint32_t z = 0, uint32_t w = 0) { return x | (y << 3u) | (z << 6u) | (w << 9u); } @@ -87,9 +75,7 @@ struct ShaderVertexInputInfo { int fetch_attrib_reg = 0; int fetch_buffer_reg = 0; int buffers_num = 0; - int export_count = 0; uint32_t scratch_size_dwords = 0; - uint32_t param_export_mask = 0; uint32_t pa_cl_vs_out_cntl = 0; ShaderClipSpaceTransform clip_space; bool fetch_external = false; @@ -119,8 +105,6 @@ struct ShaderPixelInputInfo { uint32_t ps_perspective_center_vgpr = UINT32_MAX; uint8_t target_output_mode[8] = {}; std::array target_export_mapping = {}; - uint32_t mrt_output_mask = 0; - uint32_t push_constant_offset = 0; uint32_t scratch_size_dwords = 0; bool ps_pos_x = false; bool ps_pos_y = false; diff --git a/src/graphics/shader/shaderCompiler.h b/src/graphics/shader/shaderCompiler.h index 559912d53..4859a9fee 100644 --- a/src/graphics/shader/shaderCompiler.h +++ b/src/graphics/shader/shaderCompiler.h @@ -1,10 +1,8 @@ #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_SHADERCOMPILER_H_ #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_SHADERCOMPILER_H_ -#include "graphics/host_gpu/vulkanCommon.h" #include "graphics/shader/shader.h" -#include #include #include @@ -28,26 +26,11 @@ ShaderParams PrepareProgram(const HW::VertexShaderInfo& regs, const HW::ShaderRe ShaderVertexInputInfo& input_info); ShaderParams PrepareProgram( const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh, - const ShaderVertexInputInfo& vertex_info, std::span target_export_mapping, ShaderPixelInputInfo& input_info); ShaderParams PrepareProgram(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh, ShaderComputeInputInfo& input_info); -bool MaterializeProgram(const std::shared_ptr& program, - const ShaderParams& params, ShaderVertexInputInfo& input_info); -bool MaterializeProgram(const std::shared_ptr& program, - const ShaderParams& params, ShaderPixelInputInfo& input_info); -bool MaterializeProgram(const std::shared_ptr& program, - const ShaderParams& params, ShaderComputeInputInfo& input_info); - -vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, - ShaderVertexInputInfo& input_info); -vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, - ShaderPixelInputInfo& input_info); -vk::ShaderModule CompileProgram(vk::Device device, const ShaderParams& params, - ShaderComputeInputInfo& input_info); - } // namespace Libs::Graphics #endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_SHADERCOMPILER_H_ */ diff --git a/src/graphics/shader/shaderStageRuntime.cpp b/src/graphics/shader/shaderStageRuntime.cpp deleted file mode 100644 index d1ff489de..000000000 --- a/src/graphics/shader/shaderStageRuntime.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" -#include "graphics/shader/shader.h" - -#include - -namespace Libs::Graphics { - -bool ShaderMaterializeStageRuntime(std::shared_ptr program, - std::span user_data, uint64_t shader_base, - ShaderStageRuntime& stage, - ShaderSpecializationMemoryReader read_specialization_memory, - void* read_memory_data) { - if (program == nullptr) { - return false; - } - ShaderRecompiler::IR::SrtRuntime runtime; - runtime.user_data = user_data; - runtime.shader_base = shader_base; - runtime.read_specialization_memory = read_specialization_memory; - runtime.userdata = read_memory_data; - ShaderRecompiler::IR::ResourceSnapshot snapshot; - if (!ShaderRecompiler::IR::MaterializeResources(*program, runtime, snapshot) || - !ShaderRecompiler::IR::ValidateResourceSpecialization(*program, snapshot)) { - return false; - } - auto resources = - std::make_shared(std::move(snapshot)); - stage = {std::move(program), std::move(resources)}; - return true; -} - -} // namespace Libs::Graphics diff --git a/tests/ResourceMaterializationTests.cpp b/tests/ResourceMaterializationTests.cpp new file mode 100644 index 000000000..fd2a0118a --- /dev/null +++ b/tests/ResourceMaterializationTests.cpp @@ -0,0 +1,233 @@ +#include "graphics/shader/recompiler/ir/ShaderIR.h" +#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" + +#include +#include +#include + +namespace { + +void Check(bool value, const char *text) { + if (!value) { + std::fprintf(stderr, "ResourceMaterializationTests: failed: %s\n", text); + std::abort(); + } +} + +bool RejectSpecializationRead(void *userdata, uint64_t, uint32_t *) { + ++*static_cast(userdata); + return false; +} + +Libs::Graphics::ShaderRecompiler::IR::Block & +AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + auto block = std::make_unique(); + auto *result = block.get(); + program.blocks.push_back(result); + program.block_info.push_back({.id = 0}); + program.block_storage.push_back(std::move(block)); + return *result; +} + +Libs::Graphics::ShaderRecompiler::IR::ResourcePlan SrtPlan(uint64_t address) { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + auto &value_block = AddValueBlock(program); + + MemoryInfo memory; + memory.kind = ResourceKind::ScalarAddress; + memory.planning_only = true; + program.memory_info.push_back(memory); + const auto low = Value(static_cast(address)); + const auto high = Value(static_cast(address >> 32u)); + auto &handle = + value_block.AppendNewInst(ValueOpcode::GetAddressResource, {low, high}); + auto &raw = value_block.AppendNewInst( + ValueOpcode::LoadAddressU32, + {Value(&handle), Value(0u), Value(0u), Value(true)}); + raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40}); + program.srt_reads.push_back({Value(&raw), 0}); + + auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource); + auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst, + {Value(&srt), Value(0u)}); + DescriptorSource source; + source.dwords[0] = Value(&flat); + source.dwords[1] = Value(0u); + source.dword_count = 2; + program.descriptor_sources.push_back(source); + return ExtractResourcePlan(program); +} + +Libs::Graphics::ShaderRecompiler::IR::ResourcePlan UnbasedFlatPlan() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + AddValueBlock(program); + program.info.uses_dma = true; + return ExtractResourcePlan(program); +} + +Libs::Graphics::ShaderRecompiler::IR::ResourcePlan UserDataBufferPlan() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + auto &value_block = AddValueBlock(program); + + auto &user_data = value_block.AppendNewInst( + ValueOpcode::GetUserData, {Value(static_cast(0))}); + DescriptorSource source; + source.dwords[0] = Value(&user_data); + source.dwords[1] = Value(0u); + source.dwords[2] = Value(0u); + source.dwords[3] = Value(0u); + source.dword_count = 4; + program.descriptor_sources.push_back(source); + program.info.buffers.push_back({.source = 0}); + return ExtractResourcePlan(program); +} + +Libs::Graphics::ShaderRecompiler::IR::ResourcePlan MixedSamplerPlan() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + Program program; + program.stage = Libs::Graphics::ShaderType::Compute; + program.srt_plan_complete = true; + program.resource_tracking_complete = true; + AddValueBlock(program); + + const auto AddSource = [&program](uint32_t dword_count, uint32_t first) { + DescriptorSource source; + source.dword_count = dword_count; + source.dwords[0] = Value(first); + for (uint32_t i = 1; i < dword_count; i++) { + source.dwords[i] = Value(0u); + } + program.descriptor_sources.push_back(source); + return static_cast(program.descriptor_sources.size() - 1u); + }; + + const auto image0 = AddSource(8, 0); + const auto image1 = AddSource(8, 0); + const auto sampler0 = AddSource(4, 0x11111111u); + const auto sampler1 = AddSource(4, 0x22222222u); + program.info.images.push_back( + {.source = image0, + .resource_class = ImageResourceClass::Sampled, + .numeric_class = Libs::Graphics::Prospero::TextureNumericClass::Float, + .dimension = + Libs::Graphics::ShaderRecompiler::Decoder::ImageDimension::Dim2D}); + program.info.images.push_back( + {.source = image1, + .resource_class = ImageResourceClass::Sampled, + .numeric_class = Libs::Graphics::Prospero::TextureNumericClass::Float, + .dimension = + Libs::Graphics::ShaderRecompiler::Decoder::ImageDimension::Dim2D, + .conversion_format = + Libs::Graphics::Prospero::BufferFormat::k8_8_8_8UNorm}); + program.info.samplers.push_back({.source = sampler0}); + program.info.samplers.push_back({.source = sampler1}); + program.info.sampled_pairs.push_back({.image = 0, .sampler = 0}); + program.info.sampled_pairs.push_back({.image = 0, .sampler = 1}); + program.info.sampled_pairs.push_back({.image = 1, .sampler = 1}); + return ExtractResourcePlan(program); +} + +void TestMappedSrtUsesDirectReaderByDefault() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + const uint32_t dword = 0x12345678; + auto plan = SrtPlan(reinterpret_cast(&dword)); + uint32_t specialization_reads = 0; + const SrtRuntime runtime{.userdata = &specialization_reads, + .read_specialization_memory = + RejectSpecializationRead}; + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, runtime, snapshot, specialization), + "mapped SRT stage materialization failed"); + Check(specialization_reads == 0, + "ordinary SRT read used the specialization reader"); + Check(snapshot.flattened_srt.size() == 1 && + snapshot.flattened_srt[0] == dword, + "cache rematerialization did not use the direct reader by default"); +} + +void TestUnbasedFlatCacheHitMaterializes() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + auto plan = UnbasedFlatPlan(); + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, {}, snapshot, specialization), + "unbased FLAT stage materialization failed"); + Check(snapshot.buffers.empty() && snapshot.images.empty(), + "unbased FLAT plan produced unexpected descriptors"); +} + +void TestFailedMaterializationPreservesPriorStage() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + auto plan = UserDataBufferPlan(); + ResourceSnapshot snapshot; + snapshot.user_data.push_back(0xfeedbeefu); + ResourceSpecialization specialization; + specialization.buffers.push_back({.packed_stride = 7}); + Check(!MaterializeResources(plan, {}, snapshot, specialization), + "missing runtime user data did not reject the cached stage"); + Check(snapshot.user_data == std::vector{0xfeedbeefu} && + specialization.buffers.size() == 1 && + specialization.buffers[0].packed_stride == 7, + "failed cache materialization changed its destinations"); +} + +void TestMixedSamplerDuplicatesTheCorrectSnapshot() { + using namespace Libs::Graphics::ShaderRecompiler::IR; + auto plan = MixedSamplerPlan(); + ResourceSnapshot snapshot; + ResourceSpecialization specialization; + Check(MaterializeResources(plan, {}, snapshot, specialization), + "mixed sampler materialization failed"); + Check(snapshot.samplers.size() == 3, + "mixed sampler materialization appended unrelated samplers"); + Check(snapshot.samplers[2] == snapshot.samplers[1] && + snapshot.samplers[2] != snapshot.samplers[0], + "point sampler variant duplicated the wrong runtime descriptor"); +} + +} // namespace + +namespace Common { + +int DbgExitHandler(const char *, int, std::string_view) { std::abort(); } + +int DbgExitHandler(const char *, int, fmt::text_style, std::string_view) { + std::abort(); +} + +int DbgExitIfHandler(const char *, const char *, int) { return 1; } + +void DbgExit(int) { std::abort(); } + +} // namespace Common + +int main() { + TestMappedSrtUsesDirectReaderByDefault(); + TestUnbasedFlatCacheHitMaterializes(); + TestFailedMaterializationPreservesPriorStage(); + TestMixedSamplerDuplicatesTheCorrectSnapshot(); + std::puts("ResourceMaterializationTests: all cases passed"); + return 0; +} + +// Keep this focused standalone target self-contained by amalgamating its small +// typed-IR implementation set. +#include "graphics/shader/recompiler/ir/Block.cpp" +#include "graphics/shader/recompiler/ir/Program.cpp" +#include "graphics/shader/recompiler/ir/Type.cpp" +#include "graphics/shader/recompiler/ir/Value.cpp" +#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index e7e283020..f48762ee4 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -8,6 +8,7 @@ #include "graphics/shader/recompiler/ir/passes/SrtWalker.h" #include +#include #include #include #include @@ -29,6 +30,14 @@ void Check(bool condition, const char *message) { } } +bool SameResourceSnapshot(const ResourceSnapshot &lhs, + const ResourceSnapshot &rhs) { + return lhs.buffers == rhs.buffers && lhs.images == rhs.images && + lhs.samplers == rhs.samplers && + lhs.flattened_srt == rhs.flattened_srt && + lhs.user_data == rhs.user_data; +} + template void CheckFatal(F &&function, std::string_view expected, const char *message) { try { @@ -250,6 +259,7 @@ MakeIndirectImageFixture(bool malformed, uint32_t material_immediate = 0, void TestInvariantIndirectImageMaterialization() { auto fixture = MakeIndirectImageFixture(false); fixture->PlanAndTrack(); + auto resource_plan = ExtractResourcePlan(fixture->program); EliminateDeadCode(fixture->program.blocks); ValidateProgram(fixture->program, true); @@ -298,24 +308,20 @@ void TestInvariantIndirectImageMaterialization() { .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; ResourceSnapshot snapshot; - const auto same_snapshot = [](const ResourceSnapshot &lhs, - const ResourceSnapshot &rhs) { - return lhs.buffers == rhs.buffers && lhs.images == rhs.images && - lhs.samplers == rhs.samplers && - lhs.flattened_srt == rhs.flattened_srt && - lhs.user_data == rhs.user_data && - lhs.indirect_images.empty() == rhs.indirect_images.empty(); - }; - Check(MaterializeResources(fixture->program, runtime, snapshot) && + ResourceSpecialization specialization; + Check(MaterializeResources(resource_plan, runtime, snapshot, specialization) && snapshot.images.size() == 1 && std::equal(image_descriptor.begin(), image_descriptor.end(), snapshot.images[0].dwords.begin()), "invariant indirect image table did not materialize"); const auto prior_snapshot = snapshot; + const auto prior_specialization = specialization; memory.fail_address = 0x1004u; - Check(!MaterializeResources(fixture->program, runtime, snapshot) && - same_snapshot(snapshot, prior_snapshot), + Check(!MaterializeResources(resource_plan, runtime, snapshot, + specialization) && + SameResourceSnapshot(snapshot, prior_snapshot) && + specialization == prior_specialization, "rejected planning memory read mutated the snapshot"); memory.fail_address = UINT64_MAX; @@ -330,8 +336,9 @@ void TestInvariantIndirectImageMaterialization() { memory.words[(0x2020u - memory.base) / 4u + 3u] = image_descriptor[3] ^ (1u << 28u); ResourceSnapshot null_snapshot; - Check(MaterializeResources(fixture->program, runtime, null_snapshot) && - null_snapshot.indirect_images.empty() && + ResourceSpecialization null_specialization; + Check(MaterializeResources(resource_plan, runtime, null_snapshot, + null_specialization) && std::ranges::all_of(null_snapshot.images[0].dwords, [](uint32_t dword) { return dword == 0u; }), "stale typed null image descriptors were not canonicalized"); @@ -345,19 +352,25 @@ void TestInvariantIndirectImageMaterialization() { memory.words[(0x2020u - memory.base) / 4u] ^= 1u; memory.words[(0x1000u - memory.base + 36u) / 4u] = 1u; ResourceSnapshot dynamic_snapshot; - Check(MaterializeResources(fixture->program, runtime, dynamic_snapshot) && - dynamic_snapshot.images.size() == 1 && - dynamic_snapshot.indirect_images.size() == 1 && - dynamic_snapshot.indirect_images[0].descriptors.size() == 2, + ResourceSpecialization dynamic_specialization; + Check(MaterializeResources(resource_plan, runtime, dynamic_snapshot, + dynamic_specialization) && + dynamic_snapshot.images.size() == 2 && + dynamic_specialization.images.size() == 2, "dynamic indirect image table did not materialize"); - SpecializeResources(fixture->program, dynamic_snapshot); + ApplyResourceSpecialization(fixture->program, dynamic_specialization); Check(fixture->program.info.images.size() == 2 && fixture->program.info.images[0].indirect_root == 0 && - fixture->program.info.images[0].indirect_mapping_capacity != 0 && + fixture->program.info.images[0].indirect_search_iterations != 0 && fixture->program.info.images[0].indirect_resources.size() == 2 && - dynamic_snapshot.images.size() == 2 && - dynamic_snapshot.indirect_images.empty(), + dynamic_snapshot.images.size() == 2, "dynamic indirect image table was not specialized transactionally"); + const auto &mapping = dynamic_specialization.images[0]; + const auto key_count = dynamic_snapshot.flattened_srt[mapping.indirect_mapping_offset]; + Check(mapping.indirect_search_iterations == std::bit_width(key_count) && + mapping.indirect_mapping_offset + 1u + key_count * 2u == + dynamic_snapshot.flattened_srt.size(), + "indirect image mapping retained worst-case padding"); for (uint32_t dword = 0; dword < image_descriptor.size(); dword++) { memory.words[(0x2000u - memory.base) / 4u + dword] = @@ -368,22 +381,25 @@ void TestInvariantIndirectImageMaterialization() { memory.words[(0x2000u - memory.base) / 4u] += 0x100u; memory.words[(0x2020u - memory.base) / 4u] += 0x101u; ResourceSnapshot rebound_snapshot; - Check(MaterializeResources(fixture->program, runtime, rebound_snapshot) && - ValidateResourceSpecialization(fixture->program, rebound_snapshot), + ResourceSpecialization rebound_specialization; + Check(MaterializeResources(resource_plan, runtime, rebound_snapshot, + rebound_specialization) && + rebound_specialization == dynamic_specialization, "stable indirect key mapping did not accept changed image addresses"); memory.words[(0x2020u - memory.base) / 4u] = memory.words[(0x2000u - memory.base) / 4u]; - Check(MaterializeResources(fixture->program, runtime, rebound_snapshot) && - ValidateResourceSpecialization(fixture->program, rebound_snapshot), - "runtime indirect key mapping did not accept collapsed candidates"); - const auto collapsed_snapshot = rebound_snapshot; + Check(MaterializeResources(resource_plan, runtime, rebound_snapshot, + rebound_specialization) && + rebound_specialization != dynamic_specialization, + "collapsed indirect candidates did not select a new specialization"); + const auto collapsed_specialization = rebound_specialization; ResourceSnapshot capacity_snapshot; + ResourceSpecialization capacity_specialization; for (const uint32_t records : {1u, 3u}) { user_data[2] = records; - Check( - MaterializeResources(fixture->program, runtime, capacity_snapshot) && - ValidateResourceSpecialization(fixture->program, capacity_snapshot), - "runtime indirect key mapping rejected a fitting material-table size"); + Check(MaterializeResources(resource_plan, runtime, capacity_snapshot, + capacity_specialization), + "runtime indirect key mapping rejected a valid material-table size"); } user_data[2] = 2u; memory.words[(0x2020u - memory.base) / 4u] = @@ -395,13 +411,14 @@ void TestInvariantIndirectImageMaterialization() { image_descriptor[dword]; } memory.words[(0x1000u - memory.base + 68u) / 4u] = 2u; - Check( - !MaterializeResources(fixture->program, runtime, rebound_snapshot) && - same_snapshot(rebound_snapshot, collapsed_snapshot), - "larger indirect candidate topology reused or mutated a cached snapshot"); + Check(MaterializeResources(resource_plan, runtime, rebound_snapshot, + rebound_specialization) && + rebound_specialization != collapsed_specialization, + "larger indirect candidate topology reused the old specialization"); auto memory_backed = MakeIndirectImageFixture(false, 0u, true); memory_backed->PlanAndTrack(); + auto memory_backed_plan = ExtractResourcePlan(memory_backed->program); EliminateDeadCode(memory_backed->program.blocks); std::array memory_backed_user_data{0x1000u, 224u << 16u, 2u, 0u, 0x2000u, 16u << 16u, 4u, 0u, @@ -413,9 +430,12 @@ void TestInvariantIndirectImageMaterialization() { .userdata = &memory, .read_specialization_memory = ReadLinearTestMemory}; - Check(!MaterializeResources(memory_backed->program, memory_backed_runtime, - snapshot) && - same_snapshot(snapshot, prior_snapshot), + const auto memory_backed_prior_snapshot = snapshot; + const auto memory_backed_prior_specialization = specialization; + Check(!MaterializeResources(memory_backed_plan, memory_backed_runtime, + snapshot, specialization) && + SameResourceSnapshot(snapshot, memory_backed_prior_snapshot) && + specialization == memory_backed_prior_specialization, "rejected indirect table descriptor read mutated the snapshot"); memory.fail_address = UINT64_MAX; @@ -728,6 +748,7 @@ void TestDynamicStorageMipTracking() { const auto mip2 = AddStore(12, true, Value(2u)); const auto dynamic = AddStore(16, true, fixture.UserData(8)); fixture.PlanAndTrack(); + auto resource_plan = ExtractResourcePlan(fixture.program); const auto &images = fixture.program.info.images; Check(images.size() == 2 && images[0].mip_mode == ImageMipMode::None && @@ -758,15 +779,23 @@ void TestDynamicStorageMipTracking() { << 28u); descriptor.dwords[5] = 3u << 4u; descriptor.dword_count = 8; + std::array user_data{}; + std::copy(descriptor.dwords.begin(), descriptor.dwords.end(), + user_data.begin()); + user_data[8] = 2u; + SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; - snapshot.images.assign(images.size(), descriptor); - SpecializeResources(fixture.program, snapshot); + ResourceSpecialization specialization; + Check(MaterializeResources(resource_plan, runtime, snapshot, + specialization), + "dynamic storage resources did not materialize"); + ApplyResourceSpecialization(fixture.program, specialization); Check(fixture.program.info.images[1].mip_count == 3 && - ValidateResourceSpecialization(fixture.program, snapshot), + snapshot.images.size() == fixture.program.info.images.size(), "base-1 through last-3 dynamic storage range was not specialized"); ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program, 0); + AllocateBindings(fixture.program); const auto storage_kind = DescriptorBindingForImage(images[0]); Check(storage_kind.has_value(), "storage image has no descriptor binding"); const auto *storage_binding = @@ -775,32 +804,58 @@ void TestDynamicStorageMipTracking() { storage_binding->resources == std::vector({0, 1, 1, 1}), "dynamic storage mip descriptors were not expanded consecutively"); - Program null_program; - null_program.resource_tracking_complete = true; - ImageResource null_image; - null_image.resource_class = ImageResourceClass::Storage; - null_image.dimension = Decoder::ImageDimension::Dim2D; - null_image.mip_mode = ImageMipMode::DynamicStorage; - null_image.written = true; - null_program.info.images.push_back(null_image); + Fixture null_fixture; + const auto null_handle = null_fixture.Image( + {Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u)}); + const auto null_address = null_fixture.Emit( + ValueOpcode::MakeImageAddress, + {Value(0u), Value(0u), null_fixture.UserData(0), Value(0u), Value(0u), + Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), Value(0u), + Value(0u), Value(0u)}); + MemoryInfo null_memory; + null_memory.kind = ResourceKind::Image; + null_memory.image_dimension = Decoder::ImageDimension::Dim2D; + null_memory.image_address_components = 3u; + null_memory.image_has_mip = true; + const auto null_data = null_fixture.Emit( + ValueOpcode::CompositeConstructU32x4, + {Value(1u), Value(2u), Value(3u), Value(4u)}); + null_fixture.Emit(ValueOpcode::ImageWrite, + {null_handle, null_address, null_data, Value(true)}, + null_fixture.AddMemory(null_memory, 4)); + null_fixture.PlanAndTrack(); + auto null_plan = ExtractResourcePlan(null_fixture.program); ResourceSnapshot null_snapshot; - DescriptorValue null_descriptor{}; - null_descriptor.dword_count = 8; - null_snapshot.images.push_back(null_descriptor); - SpecializeResources(null_program, null_snapshot); - Check(null_program.info.images[0].mip_count == 1 && - ValidateResourceSpecialization(null_program, null_snapshot), + ResourceSpecialization null_specialization; + const std::array null_user_data{0u}; + Check(MaterializeResources(null_plan, {.user_data = null_user_data}, + null_snapshot, null_specialization), + "canonical null dynamic storage image did not materialize"); + ApplyResourceSpecialization(null_fixture.program, null_specialization); + Check(null_fixture.program.info.images[0].mip_count == 1 && + null_snapshot.images.size() == 1, "canonical null dynamic storage image did not retain one descriptor"); - snapshot.images[1].dwords[3] = - (snapshot.images[1].dwords[3] & ~(0xfu << 16u)) | (2u << 16u); - Check(!ValidateResourceSpecialization(fixture.program, snapshot), - "a changed dynamic storage mip count reused the specialization"); - snapshot.images[1].dwords[3] = - (snapshot.images[1].dwords[3] & ~((0xfu << 12u) | (0xfu << 16u))) | + auto changed_user_data = user_data; + changed_user_data[3] = + (changed_user_data[3] & ~(0xfu << 16u)) | (2u << 16u); + ResourceSnapshot changed_snapshot; + ResourceSpecialization changed_specialization; + Check(MaterializeResources(resource_plan, {.user_data = changed_user_data}, + changed_snapshot, changed_specialization) && + changed_specialization != specialization, + "a changed dynamic storage mip count reused the specialization key"); + changed_user_data[3] = + (changed_user_data[3] & ~((0xfu << 12u) | (0xfu << 16u))) | (4u << 12u) | (3u << 16u); - Check(!ValidateResourceSpecialization(fixture.program, snapshot), - "an inverted dynamic storage mip range was accepted"); + const auto valid_snapshot = changed_snapshot; + const auto valid_specialization = changed_specialization; + Check(!MaterializeResources(resource_plan, {.user_data = changed_user_data}, + changed_snapshot, changed_specialization) && + SameResourceSnapshot(changed_snapshot, valid_snapshot) && + changed_specialization == valid_specialization, + "an inverted dynamic storage mip range was accepted or mutated output"); } void TestSrtFlatteningAndRuntimeMemoization() { @@ -843,7 +898,7 @@ void TestSrtFlatteningAndRuntimeMemoization() { .userdata = &memory}; std::vector descriptors; std::vector flat; - const DescriptorSourceRequest request{fixture.program.info.buffers[0].source}; + const uint32_t request = fixture.program.info.buffers[0].source; Check(EvaluateRuntimeSources(fixture.program, std::span{&request, 1}, runtime, descriptors, flat, {}), "typed runtime source evaluation failed"); @@ -863,7 +918,7 @@ void TestSrtFlatteningAndRuntimeMemoization() { ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program, 0); + AllocateBindings(fixture.program); Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::FlattenedSrt) != nullptr, "flattened typed SRT reads did not receive a binding"); @@ -906,7 +961,7 @@ void TestDynamicSrtReadRemainsExplicit() { ShaderComputeInputInfo compute{}; CollectShaderInfo(fixture.program, {.compute = &compute}); - AllocateBindings(fixture.program, 0); + AllocateBindings(fixture.program); Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::FlattenedSrt) == nullptr && FindBinding(fixture.program.bindings, @@ -1023,15 +1078,18 @@ void TestDmaAddressMaterialization() { {unbased, Value(0u), Value(0u), Value(9u), Value(true)}, fixture.AddMemory(flat, 8)); fixture.PlanAndTrack(); + auto resource_plan = ExtractResourcePlan(fixture.program); Check(fixture.program.info.uses_dma, "typed address operations did not enable DMA"); std::array user_data{0x2008u, 0u}; SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; - Check(MaterializeResources(fixture.program, runtime, snapshot), + ResourceSpecialization specialization; + Check(MaterializeResources(resource_plan, runtime, snapshot, + specialization), "DMA shader resources did not materialize"); - SpecializeResources(fixture.program, snapshot); + ApplyResourceSpecialization(fixture.program, specialization); } void TestDynamicFlatAddressesUseDma() { @@ -1053,13 +1111,16 @@ void TestDynamicFlatAddressesUseDma() { fixture.Emit(ValueOpcode::LoadAddressU8, {address, low, high, active}, fixture.AddMemory(flat, 0xa4)); fixture.PlanAndTrack(); + auto resource_plan = ExtractResourcePlan(fixture.program); Check(fixture.program.info.uses_dma, "exec-masked FLAT address did not enable DMA"); std::array user_data{0x23456780u, 1u, 1u}; SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; - Check(MaterializeResources(fixture.program, runtime, snapshot), + ResourceSpecialization specialization; + Check(MaterializeResources(resource_plan, runtime, snapshot, + specialization), "exec-masked FLAT shader resources did not materialize"); Fixture mismatch; @@ -1095,6 +1156,7 @@ void TestBufferSwizzleSpecialization() { {handle, Value(0u), Value(0u), Value(0u), Value(true)}, fixture.AddMemory(memory, 4)); fixture.PlanAndTrack(); + auto resource_plan = ExtractResourcePlan(fixture.program); constexpr auto swizzle = Libs::Graphics::DstSel(4, 5, 0, 1); std::array user_data{ @@ -1106,16 +1168,22 @@ void TestBufferSwizzleSpecialization() { (1u << 24u)}; SrtRuntime runtime{.user_data = user_data}; ResourceSnapshot snapshot; - Check(MaterializeResources(fixture.program, runtime, snapshot), + ResourceSpecialization specialization; + Check(MaterializeResources(resource_plan, runtime, snapshot, + specialization), "buffer resources did not materialize"); - SpecializeResources(fixture.program, snapshot); + ApplyResourceSpecialization(fixture.program, specialization); Check(fixture.program.info.buffers[0].descriptor_swizzle == swizzle && - ValidateResourceSpecialization(fixture.program, snapshot), + specialization.buffers[0].descriptor_swizzle == swizzle, "buffer destination selectors were not specialized"); - snapshot.buffers[0].dwords[3] ^= 1u << 9u; - Check(!ValidateResourceSpecialization(fixture.program, snapshot), - "buffer swizzle change did not invalidate specialization"); + user_data[3] ^= 1u << 9u; + ResourceSnapshot changed_snapshot; + ResourceSpecialization changed_specialization; + Check(MaterializeResources(resource_plan, runtime, changed_snapshot, + changed_specialization) && + changed_specialization != specialization, + "buffer swizzle change did not select a new specialization key"); } void TestShaderInfoAndBindingLayout() { @@ -1147,15 +1215,14 @@ void TestShaderInfoAndBindingLayout() { StageInputKind::GlobalInvocationId, "typed shader values were not reflected in shader info"); - AllocateBindings(fixture.program, 0); + AllocateBindings(fixture.program); Check(FindBinding(fixture.program.bindings, DescriptorBindingKind::Buffers) != nullptr && FindBinding(fixture.program.bindings, DescriptorBindingKind::Gds) != nullptr && FindBinding(fixture.program.bindings, - DescriptorBindingKind::UserData) == nullptr && - fixture.program.bindings.push_constant_size == - fixture.program.bindings.ShaderDataDwords() * sizeof(uint32_t), + DescriptorBindingKind::ShaderData) == nullptr && + fixture.program.bindings.UsesPushData(), "typed resources were not assigned native bindings"); Check(NativeBinding(ShaderType::Compute, DescriptorBindingKind::Buffers) == static_cast(DescriptorBindingKind::Buffers) && @@ -1180,7 +1247,7 @@ void TestImageBindingAbi() { static_cast(DescriptorBindingKind::BdaPagetable) == 39u && static_cast(DescriptorBindingKind::FaultBuffer) == 40u && static_cast(DescriptorBindingKind::FlattenedSrt) == 41u && - static_cast(DescriptorBindingKind::UserData) == 42u && + static_cast(DescriptorBindingKind::ShaderData) == 42u && static_cast(DescriptorBindingKind::Count) == 43u, "native descriptor binding anchors changed"); @@ -1284,58 +1351,53 @@ void TestGraphicsPushConstantLayout() { } fixture.program.shader_info_complete = true; }; - Fixture vertex(ShaderType::Vertex); - AddUserData(vertex, 9); - AllocateBindings(vertex.program, 0); - Check(vertex.program.bindings.push_constant_offset == 0 && - vertex.program.bindings.push_constant_size == 9 * sizeof(uint32_t), - "vertex shader was not placed at the start of the graphics push bank"); - - const auto pixel_offset = vertex.program.bindings.push_constant_size; + uint32_t cursor = 0; Fixture pixel(ShaderType::Pixel); AddUserData(pixel, 4); - AllocateBindings(pixel.program, pixel_offset); + AllocateBindings(pixel.program, cursor); Check( - pixel.program.bindings.push_constant_offset == pixel_offset && - pixel.program.bindings.push_constant_size == 4 * sizeof(uint32_t) && + pixel.program.bindings.UsesPushData() && + pixel.program.bindings.push_data_start_dword == 0 && FindBinding(pixel.program.bindings, - DescriptorBindingKind::UserData) == nullptr, - "pixel shader did not follow the vertex data in the graphics push bank"); + DescriptorBindingKind::ShaderData) == nullptr, + "pixel shader did not start the shared push-data block"); + pixel.program.bindings.AdvancePushData(cursor); + + Fixture vertex(ShaderType::Vertex); + AddUserData(vertex, 9); + AllocateBindings(vertex.program, cursor); + Check(vertex.program.bindings.UsesPushData() && + vertex.program.bindings.push_data_start_dword == 4, + "vertex shader did not follow pixel data in the shared push-data block"); + vertex.program.bindings.AdvancePushData(cursor); + Check(cursor == 13, "graphics push-data cursor advanced incorrectly"); Fixture edge(ShaderType::Pixel); - AddUserData(edge, 1); - AllocateBindings(edge.program, NativePushConstantSize - sizeof(uint32_t)); - Check(edge.program.bindings.push_constant_size == sizeof(uint32_t), - "last aligned push-constant dword did not fit in the graphics bank"); + AddUserData(edge, NativePushConstantSize / sizeof(uint32_t)); + AllocateBindings(edge.program); + Check(edge.program.bindings.UsesPushData() && + FindBinding(edge.program.bindings, + DescriptorBindingKind::ShaderData) == nullptr, + "the full shared push-data block did not fit"); Fixture spill(ShaderType::Pixel); - AddUserData(spill, 32); - AllocateBindings(spill.program, pixel_offset); + AddUserData(spill, 20); + AllocateBindings(spill.program, cursor); Check( - spill.program.bindings.push_constant_size == 0 && + !spill.program.bindings.UsesPushData() && + spill.program.bindings.push_data_start_dword == PushData::NoStart && FindBinding(spill.program.bindings, - DescriptorBindingKind::UserData) != nullptr, - "pixel shader overlapping the vertex push data did not spill to storage"); + DescriptorBindingKind::ShaderData) != nullptr, + "a stage that exceeded the remaining shared push data did not spill to storage"); + const auto spill_layout = spill.program.bindings; + spill.program.bindings.AdvancePushData(cursor); + Check(cursor == 13, "a spilled stage consumed shared push-data space"); - Fixture full(ShaderType::Pixel); - AddUserData(full, 1); - AllocateBindings(full.program, NativePushConstantSize); - Check(full.program.bindings.push_constant_size == 0 && - FindBinding(full.program.bindings, - DescriptorBindingKind::UserData) != nullptr, - "full graphics push bank did not spill pixel user data to storage"); - - Fixture invalid(ShaderType::Pixel); - AddUserData(invalid, 1); - CheckFatal( - [&] { - AllocateBindings(invalid.program, - NativePushConstantSize + sizeof(uint32_t)); - }, - "push-constant offset", - "push-constant placement beyond the graphics bank was accepted"); - Check(!invalid.program.binding_layout_complete, - "push-constant placement beyond the graphics bank was accepted"); + Fixture repeated_spill(ShaderType::Pixel); + AddUserData(repeated_spill, 20); + AllocateBindings(repeated_spill.program, 20); + Check(repeated_spill.program.bindings == spill_layout, + "storage fallback retained an irrelevant attempted push-data position"); } void TestResourceLimitIsTransactional() { diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 03435ed98..f0280d80f 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -473,10 +473,6 @@ struct RenderExecutorTestAccess { } } - static const auto &PushConstants(const RenderExecutor &executor) { - return executor.m_push_constants; - } - static void ResolveRenderDepthTarget(RenderExecutor &executor, uint64_t submit_id, CommandBuffer &buffer, @@ -1041,7 +1037,7 @@ struct GraphicsCase { struct CompiledShader { std::vector spirv; ShaderRecompiler::IR::Program program; - std::vector flattened_srt; + ShaderRecompiler::IR::ResourceSnapshot resources; std::vector packed_user_data; }; @@ -1094,19 +1090,20 @@ size_t CountText(const std::string &text, const std::string &needle) { void CheckRectListShaders() { constexpr const char *name = "RectListShaders"; - auto program = std::make_shared(); - program->info.inputs.push_back( + ShaderRecompiler::IR::CompiledShaderInfo program{}; + program.info.inputs.push_back( {ShaderRecompiler::IR::StageInputKind::Parameter, 0, 4, "in_param_0"}); - program->info.inputs.push_back( + program.info.inputs.push_back( {ShaderRecompiler::IR::StageInputKind::Parameter, 1, 4, "in_param_1"}); ShaderVertexInputInfo vertex{}; - vertex.param_export_mask = 1u; + ShaderRecompiler::IR::CompiledShaderInfo vertex_program{.param_export_mask = 1u}; + vertex.stage.program = &vertex_program; ShaderPixelInputInfo pixel{}; pixel.input_num = 2; pixel.interpolator_settings[0] = 0x400u; pixel.interpolator_settings[1] = 0; - pixel.stage.program = program; + pixel.stage.program = &program; std::vector perspective_specialization; BuildStageStaticKey(pixel, perspective_specialization); pixel.ps_no_perspective = true; @@ -1223,14 +1220,28 @@ CompiledShader CompileCase(const TestCase &test) { options.dump_ir = true; options.input_info.compute = &test.compute_info; options.user_data = user_data; - options.read_memory = ReadTestMemory; - options.read_memory_data = const_cast *>(&test.initial); options.scratch_dwords = test.compute_info.scratch_size_dwords; if (test.has_compute_info) { options.wave_size = test.compute_info.wave_size; } - auto result = ShaderRecompiler::Recompile(test.code, options); + auto translated = ShaderRecompiler::TranslateProgram(test.code, options); + auto resource_plan = + ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = options.user_data, + .shader_base = reinterpret_cast(test.code.data()), + .read_memory = ReadTestMemory, + .userdata = const_cast *>(&test.initial), + }; + Require(test.name, "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization), + "translated resources could not be materialized"); + auto result = ShaderRecompiler::CompileProgram( + std::move(translated), options, specialization); for (const auto &[text, expected] : test.decoded_counts) { const auto actual = CountText(result.decoded_dump, text); Require(test.name, "decoded RDNA2", actual == expected, @@ -1278,17 +1289,17 @@ CompiledShader CompileCase(const TestCase &test) { } result.program.bindings = {}; result.program.binding_layout_complete = false; - ShaderRecompiler::IR::AllocateBindings(result.program, 0); + ShaderRecompiler::IR::AllocateBindings(result.program); const auto *shader_data = ShaderRecompiler::IR::FindBinding( result.program.bindings, - ShaderRecompiler::IR::DescriptorBindingKind::UserData); + ShaderRecompiler::IR::DescriptorBindingKind::ShaderData); Require(test.name, "binding layout", result.program.bindings.user_data_registers.size() == 33 && - result.program.bindings.push_constant_size == 0 && + !result.program.bindings.UsesPushData() && shader_data != nullptr, "oversized shader data did not use its storage fallback"); - result.spirv = ShaderRecompiler::Spirv::EmitProgram( - result.program, result.resources, options.input_info); + result.spirv = ShaderRecompiler::Spirv::EmitProgram(result.program, + options.input_info); } Require(test.name, "SPIR-V emit", !result.spirv.empty(), "recompiler returned empty SPIR-V"); @@ -1297,7 +1308,7 @@ CompiledShader CompileCase(const TestCase &test) { std::vector packed_user_data; for (const auto reg : result.program.bindings.user_data_registers) { packed_user_data.push_back( - result.resources.user_data[reg - result.program.user_data_base]); + resources.user_data[reg - result.program.user_data_base]); } packed_user_data.resize(result.program.bindings.ShaderDataDwords()); const auto buffer_count = @@ -1313,8 +1324,7 @@ CompiledShader CompileCase(const TestCase &test) { offset << ((i % 4u) * 8u); } return {std::move(result.spirv), std::move(result.program), - std::move(result.resources.flattened_srt), - std::move(packed_user_data)}; + std::move(resources), std::move(packed_user_data)}; } std::array MakeStructuredStorageBufferData(u32 stride_bytes, @@ -1364,19 +1374,33 @@ CompiledShader CompileFragmentCase(const GraphicsCase &test) { options.input_info.pixel = &pixel_info; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(test.fragment_code, options); + auto translated = + ShaderRecompiler::TranslateProgram(test.fragment_code, options); + auto resource_plan = + ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = options.user_data, + .shader_base = reinterpret_cast(test.fragment_code.data()), + }; + Require(test.name, "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization), + "translated resources could not be materialized"); + auto result = ShaderRecompiler::CompileProgram( + std::move(translated), options, specialization); Require(test.name, "SPIR-V emit", !result.spirv.empty(), "recompiler returned empty SPIR-V"); ValidateSpirv(test.name, result.spirv); std::vector packed_user_data; for (const auto reg : result.program.bindings.user_data_registers) { packed_user_data.push_back( - result.resources.user_data[reg - result.program.user_data_base]); + resources.user_data[reg - result.program.user_data_base]); } packed_user_data.resize(result.program.bindings.ShaderDataDwords()); return {std::move(result.spirv), std::move(result.program), - std::move(result.resources.flattened_srt), - std::move(packed_user_data)}; + std::move(resources), std::move(packed_user_data)}; } std::array MakeSampledTextureData(Prospero::BufferFormat format) { @@ -1591,7 +1615,7 @@ public: } void CheckGraphicsPushConstantBank() { - constexpr const char *name = "GraphicsPushConstantBank"; + constexpr const char *name = "GraphicsPushConstantStages"; EnsureRuntimeContext(); RenderContext context(m_runtime_context); auto &scheduler = context.GetCommandScheduler(); @@ -1600,34 +1624,34 @@ public: HW::Shader shaders{}; scheduler.Begin(registers, user_config, shaders); - ShaderRecompiler::IR::Program vertex_program{}; + ShaderRecompiler::IR::CompiledShaderInfo vertex_program{}; vertex_program.stage = ShaderType::Vertex; - vertex_program.bindings.push_constant_size = 2 * sizeof(uint32_t); - ShaderRecompiler::IR::Program pixel_program{}; + vertex_program.bindings.push_data_start_dword = 2; + vertex_program.bindings.user_data_registers = {0, 1}; + vertex_program.bindings.memory_offset_dword = 2; + ShaderRecompiler::IR::CompiledShaderInfo pixel_program{}; pixel_program.stage = ShaderType::Pixel; - pixel_program.bindings.push_constant_offset = 2 * sizeof(uint32_t); - pixel_program.bindings.push_constant_size = 2 * sizeof(uint32_t); + pixel_program.bindings.push_data_start_dword = 0; + pixel_program.bindings.user_data_registers = {0, 1}; + pixel_program.bindings.memory_offset_dword = 2; ShaderRecompiler::IR::ResourceSnapshot snapshot{}; PreparedBindings vertex{}; vertex.program = &vertex_program; vertex.snapshot = &snapshot; - vertex.user_data = {0x11111111u, 0x22222222u}; + vertex.shader_data = {0x11111111u, 0x22222222u}; PreparedBindings pixel{}; pixel.program = &pixel_program; pixel.snapshot = &snapshot; - pixel.user_data = {0x33333333u, 0x44444444u}; + pixel.shader_data = {0x33333333u, 0x44444444u}; const auto pipeline = RenderExecutorTestAccess::CommitBindings( context.GetRenderExecutor(), scheduler.Current(), vertex, pixel); - const auto &bank = - RenderExecutorTestAccess::PushConstants(context.GetRenderExecutor()); - Require(name, "packing", - bank[0] == 0x11111111u && bank[1] == 0x22222222u && - bank[2] == 0x33333333u && bank[3] == 0x44444444u && - std::ranges::all_of(bank.begin() + 4, bank.end(), - [](uint32_t value) { return value == 0; }) && - vertex.committed && pixel.committed, - "graphics stages were not packed into one zero-filled push bank"); + Require(name, "shared push data", + vertex.shader_data == + std::vector{0x11111111u, 0x22222222u} && + pixel.shader_data == + std::vector{0x33333333u, 0x44444444u}, + "graphics stages did not commit their shared push data"); scheduler.Finish(); RenderExecutorTestAccess::DestroyDescriptorPipelines( context.GetRenderExecutor(), std::span {&pipeline, 1u}); @@ -3838,14 +3862,14 @@ public: !texture_cache.IsMetaCleared(write_only_meta, 0), "test HTile entries were not registered in their initial state"); - const auto MakeInput = [](uint64_t address, bool read, bool written) { - auto program = std::make_shared(); - program->stage = ShaderType::Compute; + const auto MakeInput = [](uint64_t address, bool read, bool written, + ShaderRecompiler::IR::CompiledShaderInfo &program) { + program.stage = ShaderType::Compute; ShaderRecompiler::IR::BufferResource resource{}; resource.read = read; resource.written = written; resource.formatted = true; - program->info.buffers.push_back(resource); + program.info.buffers.push_back(resource); ShaderBufferResource descriptor{}; descriptor.UpdateAddress48(address); @@ -3858,18 +3882,16 @@ public: ShaderRecompiler::IR::DescriptorValue value{}; value.dword_count = 4; std::copy_n(descriptor.fields, value.dword_count, value.dwords.begin()); - auto snapshot = - std::make_shared(); - snapshot->buffers.push_back(value); - ShaderComputeInputInfo input{}; - input.stage.program = std::move(program); - input.stage.resources = std::move(snapshot); + input.stage.program = &program; + input.stage.resources.buffers.push_back(value); return input; }; auto &executor = context.GetRenderExecutor(); auto &command = scheduler.Current(); - const auto read_only_input = MakeInput(read_only_meta, true, false); + ShaderRecompiler::IR::CompiledShaderInfo read_only_program{}; + const auto read_only_input = + MakeInput(read_only_meta, true, false, read_only_program); const bool read_only_consumed = RenderExecutorTestAccess::TryConsumeComputeMetaClear( executor, read_only_input, command); @@ -3878,7 +3900,9 @@ public: !texture_cache.IsMetaCleared(read_only_meta, 0), "a metadata read-only dispatch changed logical clear state"); - const auto read_write_input = MakeInput(read_write_meta, true, true); + ShaderRecompiler::IR::CompiledShaderInfo read_write_program{}; + const auto read_write_input = + MakeInput(read_write_meta, true, true, read_write_program); const bool read_write_consumed = RenderExecutorTestAccess::TryConsumeComputeMetaClear( executor, read_write_input, command); @@ -3887,7 +3911,9 @@ public: !texture_cache.IsMetaCleared(read_write_meta, 0), "a metadata read/modify/write dispatch was replaced by a clear"); - const auto write_only_input = MakeInput(write_only_meta, false, true); + ShaderRecompiler::IR::CompiledShaderInfo write_only_program{}; + const auto write_only_input = + MakeInput(write_only_meta, false, true, write_only_program); const bool write_only_consumed = RenderExecutorTestAccess::TryConsumeComputeMetaClear( executor, write_only_input, command); @@ -7704,7 +7730,7 @@ public: const auto allocate_bindings = [&](ShaderRecompiler::IR::Program &program) { program.shader_info_complete = true; - ShaderRecompiler::IR::AllocateBindings(program, 0); + ShaderRecompiler::IR::AllocateBindings(program); }; constexpr auto stencil_format = Prospero::BufferFormat::k8UInt; @@ -7741,13 +7767,20 @@ public: ShaderRecompiler::IR::DescriptorValue null_descriptor{}; null_descriptor.dword_count = 8; null_snapshot.images.assign(3, null_descriptor); - ShaderRecompiler::IR::SpecializeResources(null_program, null_snapshot); + ShaderRecompiler::IR::ResourceSpecialization null_specialization{}; + null_specialization.images.resize(3); + for (auto &image : null_specialization.images) { + image.numeric_class = Prospero::TextureNumericClass::Float; + image.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D; + } + ShaderRecompiler::IR::ApplyResourceSpecialization( + null_program, null_specialization); allocate_bindings(null_program); - ShaderStageRuntime null_runtime{ - std::make_shared( - std::move(null_program)), - std::make_shared( - std::move(null_snapshot))}; + ShaderRecompiler::IR::CompiledShaderInfo null_info{}; + null_info.stage = null_program.stage; + null_info.info = std::move(null_program.info); + null_info.bindings = std::move(null_program.bindings); + ShaderStageRuntime null_runtime{&null_info, std::move(null_snapshot)}; auto null_bindings = executor.PrepareBindings(null_runtime); executor.RebindImages(null_bindings); Require(name, "null descriptor count", @@ -7892,19 +7925,18 @@ public: executor, sampled_overwide_resource, overwide_mipped_storage_descriptor); PreparedBindings mipped_prepared{}; - auto mipped_program = std::make_shared(); - mipped_program->info.images.push_back(storage_resource); - mipped_program->info.images.push_back(mipped_storage_resource); - mipped_program->info.images.push_back(storage_resource); - mipped_program->info.images.push_back(sampled_overwide_resource); - auto mipped_snapshot = - std::make_shared(); - mipped_snapshot->images = {mipped_storage_descriptor, - mipped_storage_descriptor, - overwide_mipped_storage_descriptor, - overwide_mipped_storage_descriptor}; - mipped_prepared.program = mipped_program.get(); - mipped_prepared.snapshot = mipped_snapshot.get(); + ShaderRecompiler::IR::CompiledShaderInfo mipped_program{}; + mipped_program.info.images.push_back(storage_resource); + mipped_program.info.images.push_back(mipped_storage_resource); + mipped_program.info.images.push_back(storage_resource); + mipped_program.info.images.push_back(sampled_overwide_resource); + ShaderRecompiler::IR::ResourceSnapshot mipped_snapshot{}; + mipped_snapshot.images = {mipped_storage_descriptor, + mipped_storage_descriptor, + overwide_mipped_storage_descriptor, + overwide_mipped_storage_descriptor}; + mipped_prepared.program = &mipped_program; + mipped_prepared.snapshot = &mipped_snapshot; mipped_prepared.resources.images.push_back( std::move(plain_mipped_storage_binding)); mipped_prepared.resources.images.push_back( @@ -8094,11 +8126,12 @@ public: ShaderRecompiler::IR::ResourceSnapshot storage_snapshot{}; storage_snapshot.images.push_back(storage_descriptor); - ShaderStageRuntime storage_runtime{ - std::make_shared( - std::move(storage_program)), - std::make_shared( - std::move(storage_snapshot))}; + ShaderRecompiler::IR::CompiledShaderInfo storage_info{}; + storage_info.stage = storage_program.stage; + storage_info.info = std::move(storage_program.info); + storage_info.bindings = std::move(storage_program.bindings); + ShaderStageRuntime storage_runtime{&storage_info, + std::move(storage_snapshot)}; auto storage_discovery = executor.PrepareBindings(storage_runtime); const auto storage_id = storage_discovery.resources.images[0].image_id; Require(name, "storage prefetch purity", @@ -8124,11 +8157,12 @@ public: allocate_bindings(sampled_program); ShaderRecompiler::IR::ResourceSnapshot sampled_snapshot{}; sampled_snapshot.images.push_back(storage_descriptor); - ShaderStageRuntime sampled_runtime{ - std::make_shared( - std::move(sampled_program)), - std::make_shared( - std::move(sampled_snapshot))}; + ShaderRecompiler::IR::CompiledShaderInfo sampled_info{}; + sampled_info.stage = sampled_program.stage; + sampled_info.info = std::move(sampled_program.info); + sampled_info.bindings = std::move(sampled_program.bindings); + ShaderStageRuntime sampled_runtime{&sampled_info, + std::move(sampled_snapshot)}; constexpr uint64_t ordered_sampled_address = base + 0x10000; const uint32_t ordered_sampled_value = 0x89abcdefu; @@ -8148,10 +8182,8 @@ public: ordered_descriptor.dword_count = 8; ShaderRecompiler::IR::ResourceSnapshot ordered_snapshot{}; ordered_snapshot.images.push_back(ordered_descriptor); - ShaderStageRuntime ordered_sampled_runtime{ - sampled_runtime.program, - std::make_shared( - std::move(ordered_snapshot))}; + ShaderStageRuntime ordered_sampled_runtime{sampled_runtime.program, + std::move(ordered_snapshot)}; auto ordered_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( executor, storage_runtime, ordered_sampled_runtime, true); const auto ordered_sampled_id = @@ -8203,21 +8235,28 @@ public: "a later sampled alias dropped an earlier storage write access"); RenderExecutorTestAccess::ResetBindings(executor); - auto vertex_sampled_program = - std::make_shared(); - vertex_sampled_program->stage = ShaderType::Vertex; - vertex_sampled_program->resource_tracking_complete = true; - vertex_sampled_program->info.images.push_back(sampled_resource); - allocate_bindings(*vertex_sampled_program); + ShaderRecompiler::IR::Program vertex_sampled_program{}; + vertex_sampled_program.stage = ShaderType::Vertex; + vertex_sampled_program.resource_tracking_complete = true; + vertex_sampled_program.info.images.push_back(sampled_resource); + allocate_bindings(vertex_sampled_program); + ShaderRecompiler::IR::CompiledShaderInfo vertex_sampled_info{}; + vertex_sampled_info.stage = vertex_sampled_program.stage; + vertex_sampled_info.info = std::move(vertex_sampled_program.info); + vertex_sampled_info.bindings = + std::move(vertex_sampled_program.bindings); ShaderStageRuntime vertex_sampled_runtime{ - std::move(vertex_sampled_program), sampled_runtime.resources}; - auto pixel_storage_program = - std::make_shared(); - pixel_storage_program->stage = ShaderType::Pixel; - pixel_storage_program->resource_tracking_complete = true; - pixel_storage_program->info.images.push_back(storage_resource); - allocate_bindings(*pixel_storage_program); - ShaderStageRuntime pixel_storage_runtime{std::move(pixel_storage_program), + &vertex_sampled_info, sampled_runtime.resources}; + ShaderRecompiler::IR::Program pixel_storage_program{}; + pixel_storage_program.stage = ShaderType::Pixel; + pixel_storage_program.resource_tracking_complete = true; + pixel_storage_program.info.images.push_back(storage_resource); + allocate_bindings(pixel_storage_program); + ShaderRecompiler::IR::CompiledShaderInfo pixel_storage_info{}; + pixel_storage_info.stage = pixel_storage_program.stage; + pixel_storage_info.info = std::move(pixel_storage_program.info); + pixel_storage_info.bindings = std::move(pixel_storage_program.bindings); + ShaderStageRuntime pixel_storage_runtime{&pixel_storage_info, storage_runtime.resources}; auto writable_alias_bindings = RenderExecutorTestAccess::PrepareGraphicsBindings( @@ -8286,16 +8325,19 @@ public: split_sampled_desc.view_info.level_count = 1; split_sampled_desc.view_info.usage = vk::ImageUsageFlagBits::eSampled; - auto split_program = std::make_shared(); - split_program->stage = ShaderType::Vertex; - split_program->resource_tracking_complete = true; - split_program->info.images = {storage_resource, sampled_resource}; - allocate_bindings(*split_program); + ShaderRecompiler::IR::Program split_ir{}; + split_ir.stage = ShaderType::Vertex; + split_ir.resource_tracking_complete = true; + split_ir.info.images = {storage_resource, sampled_resource}; + allocate_bindings(split_ir); + ShaderRecompiler::IR::CompiledShaderInfo split_program{}; + split_program.stage = split_ir.stage; + split_program.info = std::move(split_ir.info); + split_program.bindings = std::move(split_ir.bindings); PreparedBindings split_bindings{}; - auto split_snapshot = - std::make_shared(); - split_bindings.program = split_program.get(); - split_bindings.snapshot = split_snapshot.get(); + ShaderRecompiler::IR::ResourceSnapshot split_snapshot{}; + split_bindings.program = &split_program; + split_bindings.snapshot = &split_snapshot; split_bindings.resources.images.push_back( {split_id, texture_cache.FindTexture(split_id, split_storage_desc), split_storage_desc}); @@ -8670,9 +8712,8 @@ public: (static_cast(Prospero::ImageType::kColor2DArray) << 28u); array_texture.fields[4] = 1; - ShaderRecompiler::IR::Program array_program{}; + ShaderRecompiler::IR::CompiledShaderInfo array_program{}; array_program.stage = ShaderType::Compute; - array_program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource array_resource{}; array_resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled; @@ -8688,11 +8729,8 @@ public: array_descriptor.dword_count = 8; ShaderRecompiler::IR::ResourceSnapshot array_snapshot{}; array_snapshot.images.push_back(array_descriptor); - ShaderStageRuntime array_runtime{ - std::make_shared( - std::move(array_program)), - std::make_shared( - std::move(array_snapshot))}; + ShaderStageRuntime array_runtime{&array_program, + std::move(array_snapshot)}; auto array_binding = executor.PrepareBindings(array_runtime); executor.RebindImages(array_binding); @@ -8756,20 +8794,16 @@ public: std::end(colliding_msaa_texture.fields), colliding_msaa_descriptor.dwords.begin()); colliding_msaa_descriptor.dword_count = 8; - auto colliding_msaa_program = - std::make_shared(); - colliding_msaa_program->stage = ShaderType::Compute; - colliding_msaa_program->resource_tracking_complete = true; + ShaderRecompiler::IR::CompiledShaderInfo colliding_msaa_program{}; + colliding_msaa_program.stage = ShaderType::Compute; auto colliding_msaa_resource = array_resource; colliding_msaa_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; - colliding_msaa_program->info.images.push_back(colliding_msaa_resource); - auto colliding_msaa_snapshot = - std::make_shared(); - colliding_msaa_snapshot->images.push_back(colliding_msaa_descriptor); + colliding_msaa_program.info.images.push_back(colliding_msaa_resource); + ShaderRecompiler::IR::ResourceSnapshot colliding_msaa_snapshot{}; + colliding_msaa_snapshot.images.push_back(colliding_msaa_descriptor); ShaderStageRuntime colliding_msaa_runtime{ - std::move(colliding_msaa_program), - std::move(colliding_msaa_snapshot)}; + &colliding_msaa_program, std::move(colliding_msaa_snapshot)}; auto colliding_msaa_binding = executor.PrepareBindings(colliding_msaa_runtime); executor.RebindImages(colliding_msaa_binding); @@ -8820,18 +8854,15 @@ public: std::copy(std::begin(msaa_texture.fields), std::end(msaa_texture.fields), msaa_descriptor.dwords.begin()); msaa_descriptor.dword_count = 8; - auto msaa_program = std::make_shared(); - msaa_program->stage = ShaderType::Compute; - msaa_program->resource_tracking_complete = true; + ShaderRecompiler::IR::CompiledShaderInfo msaa_program{}; + msaa_program.stage = ShaderType::Compute; auto msaa_resource = array_resource; msaa_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa; - msaa_program->info.images.push_back(msaa_resource); - auto msaa_snapshot = - std::make_shared(); - msaa_snapshot->images.push_back(msaa_descriptor); - ShaderStageRuntime msaa_runtime{std::move(msaa_program), - std::move(msaa_snapshot)}; + msaa_program.info.images.push_back(msaa_resource); + ShaderRecompiler::IR::ResourceSnapshot msaa_snapshot{}; + msaa_snapshot.images.push_back(msaa_descriptor); + ShaderStageRuntime msaa_runtime{&msaa_program, std::move(msaa_snapshot)}; auto msaa_binding = executor.PrepareBindings(msaa_runtime); executor.RebindImages(msaa_binding); const auto &resolved_msaa = msaa_binding.resources.images[0]; @@ -8862,18 +8893,15 @@ public: std::end(msaa_array_texture.fields), msaa_array_descriptor.dwords.begin()); msaa_array_descriptor.dword_count = 8; - auto msaa_array_program = - std::make_shared(); - msaa_array_program->stage = ShaderType::Compute; - msaa_array_program->resource_tracking_complete = true; + ShaderRecompiler::IR::CompiledShaderInfo msaa_array_program{}; + msaa_array_program.stage = ShaderType::Compute; auto msaa_array_resource = array_resource; msaa_array_resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray; - msaa_array_program->info.images.push_back(msaa_array_resource); - auto msaa_array_snapshot = - std::make_shared(); - msaa_array_snapshot->images.push_back(msaa_array_descriptor); - ShaderStageRuntime msaa_array_runtime{std::move(msaa_array_program), + msaa_array_program.info.images.push_back(msaa_array_resource); + ShaderRecompiler::IR::ResourceSnapshot msaa_array_snapshot{}; + msaa_array_snapshot.images.push_back(msaa_array_descriptor); + ShaderStageRuntime msaa_array_runtime{&msaa_array_program, std::move(msaa_array_snapshot)}; auto msaa_array_binding = executor.PrepareBindings(msaa_array_runtime); executor.RebindImages(msaa_array_binding); @@ -9089,9 +9117,8 @@ public: DstSel(4, 5, 6, 7) | (static_cast(linear) << 20u) | (static_cast(Prospero::ImageType::kColor2D) << 28u); - ShaderRecompiler::IR::Program program{}; + ShaderRecompiler::IR::CompiledShaderInfo program{}; program.stage = ShaderType::Compute; - program.resource_tracking_complete = true; ShaderRecompiler::IR::ImageResource stencil_resource{}; stencil_resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Sampled; @@ -9107,11 +9134,7 @@ public: descriptor.dwords.begin()); descriptor.dword_count = 8; snapshot.images.push_back(descriptor); - ShaderStageRuntime runtime{ - std::make_shared( - std::move(program)), - std::make_shared( - std::move(snapshot))}; + ShaderStageRuntime runtime{&program, std::move(snapshot)}; auto prepared = context.GetRenderExecutor().PrepareBindings(runtime); const auto sampled_stencil_id = prepared.resources.images[0].image_id; @@ -9163,16 +9186,14 @@ public: "RebindImages did not acquire the associated depth owner"); RenderExecutorTestAccess::ResetBindings(executor); - auto stencil_storage_program = - std::make_shared(); - stencil_storage_program->stage = ShaderType::Compute; - stencil_storage_program->resource_tracking_complete = true; + ShaderRecompiler::IR::CompiledShaderInfo stencil_storage_program{}; + stencil_storage_program.stage = ShaderType::Compute; auto stencil_storage_resource = stencil_resource; stencil_storage_resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage; stencil_storage_resource.read = false; stencil_storage_resource.written = true; - stencil_storage_program->info.images.push_back(stencil_storage_resource); + stencil_storage_program.info.images.push_back(stencil_storage_resource); auto storage_stencil = stencil; storage_stencil.fields[5] = 0x00700000u; ShaderRecompiler::IR::DescriptorValue stencil_storage_descriptor{}; @@ -9183,9 +9204,7 @@ public: ShaderRecompiler::IR::ResourceSnapshot stencil_storage_snapshot{}; stencil_storage_snapshot.images.push_back(stencil_storage_descriptor); ShaderStageRuntime stencil_storage_runtime{ - std::move(stencil_storage_program), - std::make_shared( - std::move(stencil_storage_snapshot))}; + &stencil_storage_program, std::move(stencil_storage_snapshot)}; auto storage_redirected = executor.PrepareBindings(stencil_storage_runtime); executor.RebindImages(storage_redirected); @@ -9600,10 +9619,10 @@ public: pipeline_layout_info.setLayoutCount = 1; pipeline_layout_info.pSetLayouts = &descriptor_layout; vk::PushConstantRange push_range{}; - if (layout.push_constant_size != 0) { + if (layout.UsesPushData()) { push_range.stageFlags = vk::ShaderStageFlagBits::eCompute; push_range.offset = 0; - push_range.size = layout.push_constant_size; + push_range.size = ShaderRecompiler::IR::NativePushConstantSize; pipeline_layout_info.pushConstantRangeCount = 1; pipeline_layout_info.pPushConstantRanges = &push_range; } @@ -9736,8 +9755,9 @@ public: } if (const auto *flattened = Binding(Kind::FlattenedSrt); flattened != nullptr) { - flattened_buffer = CreateStorageBuffer(test.name, compiled.flattened_srt, - compiled.flattened_srt.size()); + flattened_buffer = + CreateStorageBuffer(test.name, compiled.resources.flattened_srt, + compiled.resources.flattened_srt.size()); flattened_info = {flattened_buffer.buffer, 0, flattened_buffer.size}; vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; @@ -9748,7 +9768,7 @@ public: write.pBufferInfo = &flattened_info; writes.push_back(write); } - if (const auto *user = Binding(Kind::UserData); user != nullptr) { + if (const auto *user = Binding(Kind::ShaderData); user != nullptr) { user_data_buffer = CreateStorageBuffer(test.name, compiled.packed_user_data, compiled.packed_user_data.size()); @@ -9756,7 +9776,7 @@ public: vk::WriteDescriptorSet write{}; write.sType = vk::StructureType::eWriteDescriptorSet; write.dstSet = descriptor_set; - write.dstBinding = Native(Kind::UserData); + write.dstBinding = Native(Kind::ShaderData); write.descriptorCount = 1; write.descriptorType = vk::DescriptorType::eStorageBuffer; write.pBufferInfo = &user_data_info; @@ -9917,14 +9937,13 @@ public: cmd.bindPipeline(vk::PipelineBindPoint::eCompute, pipeline); cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, pipeline_layout, 0, 1, &descriptor_set, 0, nullptr); - if (layout.push_constant_size != 0) { - Require(test.name, "dispatch", - compiled.packed_user_data.size() * sizeof(u32) == - layout.push_constant_size, - "native user-data size does not match push-constant range"); + if (layout.UsesPushData()) { + ShaderRecompiler::IR::PushData push_data; + std::copy(compiled.packed_user_data.begin(), + compiled.packed_user_data.end(), + push_data.dwords.begin() + layout.push_data_start_dword); cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eCompute, 0, - layout.push_constant_size, - compiled.packed_user_data.data()); + sizeof(push_data), push_data.dwords.data()); } cmd.dispatch(test.dispatch_x, test.dispatch_y, test.dispatch_z); @@ -10011,14 +10030,14 @@ public: const auto &fragment_bind = fragment.program.bindings; vk::PushConstantRange push_constant_range{}; - if (fragment_bind.push_constant_size > 0) { + if (fragment_bind.UsesPushData()) { Require(test.name, "graphics", test.push_constants.size() * sizeof(u32) == - fragment_bind.push_constant_size, + fragment_bind.ShaderDataDwords() * sizeof(u32), "fragment push constant data size does not match reflection"); push_constant_range.stageFlags = vk::ShaderStageFlagBits::eFragment; push_constant_range.offset = 0; - push_constant_range.size = fragment_bind.push_constant_size; + push_constant_range.size = ShaderRecompiler::IR::NativePushConstantSize; } vk::PipelineLayoutCreateInfo pipeline_layout_info{}; @@ -10144,9 +10163,11 @@ public: cmd.beginRendering(rendering); cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); if (push_constant_range.size != 0) { + ShaderRecompiler::IR::PushData push_data; + std::copy(test.push_constants.begin(), test.push_constants.end(), + push_data.dwords.begin() + fragment_bind.push_data_start_dword); cmd.pushConstants(pipeline_layout, vk::ShaderStageFlagBits::eFragment, 0, - fragment_bind.push_constant_size, - test.push_constants.data()); + sizeof(push_data), push_data.dwords.data()); } vk::DeviceSize offset = 0; cmd.bindVertexBuffers(0, 1, &vertex_buffer.buffer, &offset); @@ -20500,41 +20521,19 @@ void CheckIndirectImageKeySwitch() { root.read = true; root.indirect_root = 0; root.indirect_mapping_offset = 0; - root.indirect_mapping_capacity = mapping_capacity; + root.indirect_search_iterations = std::bit_width(mapping_capacity); root.indirect_resources = {0u, 1u}; auto candidate = root; - candidate.indirect_mapping_capacity = 0; + candidate.indirect_search_iterations = 0; candidate.indirect_resources.clear(); program.info.images = {root, candidate}; program.info.samplers.push_back({1u, 0x10f0u}); program.info.sampled_pairs.push_back({0u, 0u, 0x10f0u}); - AllocateBindings(program, 0); - ResourceSnapshot snapshot{}; - DescriptorValue descriptor{}; - descriptor.dword_count = 8; - descriptor.dwords[0] = 0x20u; - descriptor.dwords[1] = - static_cast(Prospero::BufferFormat::k32_32_32_32Float) << 20u; - descriptor.dwords[2] = 3u | (3u << 14u); - descriptor.dwords[3] = - DstSel(4, 5, 6, 7) | - (static_cast(Prospero::ImageType::kColor2D) << 28u); - snapshot.images = {descriptor, descriptor}; - snapshot.images[1].dwords[0] = 0x40u; - snapshot.flattened_srt.resize(1u + mapping_capacity * 2u); - snapshot.flattened_srt[0] = 2u; - snapshot.flattened_srt[1] = 0u; - snapshot.flattened_srt[2] = 0u; - snapshot.flattened_srt[3] = 1u; - snapshot.flattened_srt[4] = 1u; - DescriptorValue sampler_descriptor{}; - sampler_descriptor.dword_count = 4; - snapshot.samplers.push_back(sampler_descriptor); - + AllocateBindings(program); ShaderComputeInputInfo compute{}; ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); - auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, snapshot, + auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); ValidateSpirv(name, spirv); spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_2); @@ -21631,19 +21630,40 @@ void CheckPs5GameExampleImageClearRuntimeShape() { options.user_data = user_data; options.input_info.compute = &compute; options.dump_ir = false; - auto result = ShaderRecompiler::Recompile(code, options); + auto translated = ShaderRecompiler::TranslateProgram(code, options); + auto resource_plan = + ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = options.user_data, + .shader_base = reinterpret_cast(code.data()), + }; + Require("Ps5GameExampleImageClear", "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization), + "runtime clear resources could not be materialized"); + auto result = ShaderRecompiler::CompileProgram( + std::move(translated), options, specialization); ValidateSpirv("Ps5GameExampleImageClear", result.spirv); - return result; + return CompiledShader{std::move(result.spirv), std::move(result.program), + std::move(resources), {}}; }; const auto code = MakeCode(); auto positive = Compile(code); - compute.stage.program = std::make_shared( - std::move(positive.program)); - compute.stage.resources = - std::make_shared( - positive.resources); + ShaderRecompiler::IR::CompiledShaderInfo program{}; + program.stage = positive.program.stage; + program.shader_hash = positive.program.shader_hash; + program.wave_size = positive.program.wave_size; + program.user_data_base = positive.program.user_data_base; + program.user_data_count = positive.program.user_data_count; + program.scratch_dwords = positive.program.scratch_dwords; + program.info = std::move(positive.program.info); + program.bindings = std::move(positive.program.bindings); + compute.stage.program = &program; + compute.stage.resources = positive.resources; ShaderBufferResource descriptor{}; u32 packed_clear = 0; uint64_t size = 0; @@ -21656,16 +21676,12 @@ void CheckPs5GameExampleImageClearRuntimeShape() { auto non_repeated = positive.resources; non_repeated.user_data[7] ^= 1u; - compute.stage.resources = - std::make_shared( - non_repeated); + compute.stage.resources = non_repeated; Require("Ps5GameExampleImageClear", "non-repeated clear", !ResolveComputeImageClear(compute, 64, 1, 1, 0x61u, descriptor, packed_clear, size), "non-uniform uint4 data was replaced with a color clear"); - compute.stage.resources = - std::make_shared( - positive.resources); + compute.stage.resources = positive.resources; compute.dispatch_threads_num[0] = 32; Require("Ps5GameExampleImageClear", "partial dispatch", !ResolveComputeImageClear(compute, 32, 1, 1, 0x61u, descriptor, @@ -21727,25 +21743,37 @@ void CheckEmbeddedFetchVertexOffset() { options.user_data = user_data; options.input_info.vertex = &vertex; - auto result = ShaderRecompiler::Recompile(code, options); + auto translated = ShaderRecompiler::TranslateProgram(code, options); + auto resource_plan = + ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = options.user_data, + .shader_base = reinterpret_cast(code.data()), + }; + Require(name, "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization), + "embedded fetch resources could not be materialized"); + auto result = ShaderRecompiler::CompileProgram( + std::move(translated), options, specialization); Require(name, "fetch rewrite", result.program.info.vertex_fetch_components[0] == 4, "encoded fetch sequence was not recognized and rewritten"); - return result; + return CompiledShader{std::move(result.spirv), std::move(result.program), + std::move(resources), {}}; }; - const auto Resolve = [](const ShaderRecompiler::CompileResult &result, + const auto Resolve = [](const CompiledShader &result, u32 index_offset) { ShaderVertexInputInfo vertex; vertex.fetch_embedded = true; - auto program = std::make_shared(); - program->user_data_base = result.program.user_data_base; - program->info.vertex_offset_sgpr = - result.program.info.vertex_offset_sgpr; - vertex.stage.program = std::move(program); - vertex.stage.resources = - std::make_shared( - result.resources); + ShaderRecompiler::IR::CompiledShaderInfo program{}; + program.user_data_base = result.program.user_data_base; + program.info.vertex_offset_sgpr = result.program.info.vertex_offset_sgpr; + vertex.stage.program = &program; + vertex.stage.resources = result.resources; return ResolveVertexOffset(index_offset, vertex); }; @@ -23700,15 +23728,34 @@ void CheckImageSamplerSpecialization() { signed_image_descriptor.dwords[1] = static_cast(Prospero::BufferFormat::k32SInt) << 20u; - ShaderRecompiler::IR::ResourceSnapshot mixed_sampler_snapshot; - mixed_sampler_snapshot.images = {native_image_descriptor, - packed_image_descriptor, - signed_image_descriptor}; ShaderRecompiler::IR::DescriptorValue sampler_descriptor{}; sampler_descriptor.dword_count = 4; - mixed_sampler_snapshot.samplers.push_back(sampler_descriptor); - ShaderRecompiler::IR::SpecializeResources(mixed_sampler_program, - mixed_sampler_snapshot); + const std::array descriptors{native_image_descriptor, packed_image_descriptor, + signed_image_descriptor, sampler_descriptor}; + mixed_sampler_program.descriptor_sources.resize(descriptors.size()); + for (u32 source = 0; source < descriptors.size(); source++) { + auto &destination = mixed_sampler_program.descriptor_sources[source]; + destination.dword_count = descriptors[source].dword_count; + for (u32 dword = 0; dword < destination.dword_count; dword++) { + destination.dwords[dword] = Value(descriptors[source].dwords[dword]); + } + } + for (u32 image = 0; image < mixed_sampler_program.info.images.size(); image++) { + mixed_sampler_program.info.images[image].source = image; + } + mixed_sampler_program.info.samplers[0].source = 3; + mixed_sampler_program.srt_plan_complete = true; + auto mixed_sampler_plan = + ShaderRecompiler::IR::ExtractResourcePlan(mixed_sampler_program); + ShaderRecompiler::IR::ResourceSnapshot mixed_sampler_snapshot; + ShaderRecompiler::IR::ResourceSpecialization mixed_sampler_specialization; + Require("ImageSamplerSpecialization", "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + mixed_sampler_plan, {}, mixed_sampler_snapshot, + mixed_sampler_specialization), + "mixed sampler descriptors could not be materialized"); + ShaderRecompiler::IR::ApplyResourceSpecialization( + mixed_sampler_program, mixed_sampler_specialization); Require("ImageSamplerSpecialization", "mixed sampler variant", mixed_sampler_program.info.samplers.size() == 2u && !mixed_sampler_program.info.samplers[0].force_point_filtering && @@ -23753,7 +23800,7 @@ void CheckNativeImageDescriptorTypes() { vk::DescriptorType::eStorageBuffer && NativeDescriptorType(DescriptorBindingKind::FlattenedSrt) == vk::DescriptorType::eStorageBuffer && - NativeDescriptorType(DescriptorBindingKind::UserData) == + NativeDescriptorType(DescriptorBindingKind::ShaderData) == vk::DescriptorType::eStorageBuffer, "non-image binding has the wrong Vulkan descriptor type"); std::printf("[host] %-32s ok\n", "NativeImageDescriptorTypes"); @@ -23771,7 +23818,7 @@ void CheckShaderRecompilerFatalContracts() { (void)CompileCase(test); }); - ExpectFatal("PackedAtomicImageRejection", [] { + { ShaderRecompiler::IR::Program program; program.resource_tracking_complete = true; @@ -23794,10 +23841,23 @@ void CheckShaderRecompilerFatalContracts() { DstSel(4, 5, 6, 7) | (static_cast(Prospero::ImageType::kColor2D) << 28u); + image.source = 0; + program.info.images[0].source = 0; + program.descriptor_sources.resize(1); + program.descriptor_sources[0].dword_count = descriptor.dword_count; + for (u32 dword = 0; dword < descriptor.dword_count; dword++) { + program.descriptor_sources[0].dwords[dword] = + ShaderRecompiler::IR::Value(descriptor.dwords[dword]); + } + program.srt_plan_complete = true; + auto plan = ShaderRecompiler::IR::ExtractResourcePlan(program); ShaderRecompiler::IR::ResourceSnapshot snapshot; - snapshot.images.push_back(descriptor); - ShaderRecompiler::IR::SpecializeResources(program, snapshot); - }); + ShaderRecompiler::IR::ResourceSpecialization specialization; + Require("PackedAtomicImageRejection", "resource materialization", + !ShaderRecompiler::IR::MaterializeResources( + plan, {}, snapshot, specialization), + "packed atomic image descriptor was accepted"); + } } #endif @@ -24520,7 +24580,21 @@ void CheckEmbeddedFetchLaneSpill() { options.user_data = user_data; options.input_info.vertex = &vertex; - auto result = ShaderRecompiler::Recompile(code, options); + auto translated = ShaderRecompiler::TranslateProgram(code, options); + auto resource_plan = + ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = options.user_data, + .shader_base = reinterpret_cast(code.data()), + }; + Require("EmbeddedFetchLaneSpill", "resource materialization", + ShaderRecompiler::IR::MaterializeResources( + resource_plan, runtime, resources, specialization), + "lane-spilled fetch resources could not be materialized"); + auto result = ShaderRecompiler::CompileProgram( + std::move(translated), options, specialization); Require("EmbeddedFetchLaneSpill", "fetch rewrite", result.program.info.vertex_fetch_components[0] == 4, "lane-spilled fetch descriptor was not recognized and rewritten"); diff --git a/tests/ShaderStageRuntimeTests.cpp b/tests/ShaderStageRuntimeTests.cpp deleted file mode 100644 index f0d8e1e58..000000000 --- a/tests/ShaderStageRuntimeTests.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include "graphics/shader/recompiler/ir/ShaderIR.h" -#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h" -#include "graphics/shader/shader.h" - -#include -#include -#include - -namespace { - -void Check(bool value, const char *text) { - if (!value) { - std::fprintf(stderr, "ShaderStageRuntimeTests: failed: %s\n", text); - std::abort(); - } -} - -bool RejectSpecializationRead(void *userdata, uint64_t, uint32_t *) { - ++*static_cast(userdata); - return false; -} - -Libs::Graphics::ShaderRecompiler::IR::Block & -AddValueBlock(Libs::Graphics::ShaderRecompiler::IR::Program &program) { - using namespace Libs::Graphics::ShaderRecompiler::IR; - auto block = std::make_unique(); - auto *result = block.get(); - program.blocks.push_back(result); - program.block_info.push_back({.id = 0}); - program.block_storage.push_back(std::move(block)); - return *result; -} - -std::shared_ptr -SrtProgram(uint64_t address) { - using namespace Libs::Graphics::ShaderRecompiler::IR; - Program program; - program.stage = Libs::Graphics::ShaderType::Compute; - program.srt_plan_complete = true; - program.resource_tracking_complete = true; - auto &value_block = AddValueBlock(program); - - MemoryInfo memory; - memory.kind = ResourceKind::ScalarAddress; - memory.planning_only = true; - program.memory_info.push_back(memory); - const auto low = Value(static_cast(address)); - const auto high = Value(static_cast(address >> 32u)); - auto &handle = - value_block.AppendNewInst(ValueOpcode::GetAddressResource, {low, high}); - auto &raw = value_block.AppendNewInst( - ValueOpcode::LoadAddressU32, - {Value(&handle), Value(0u), Value(0u), Value(true)}); - raw.SetFlags(MemoryFlags{.index = 0, .pc = 0x40}); - program.srt_reads.push_back({Value(&raw), 0}); - - auto &srt = value_block.AppendNewInst(ValueOpcode::GetSrtResource); - auto &flat = value_block.AppendNewInst(ValueOpcode::ReadConst, - {Value(&srt), Value(0u)}); - DescriptorSource source; - source.dwords[0] = Value(&flat); - source.dwords[1] = Value(0u); - source.dword_count = 2; - program.descriptor_sources.push_back(source); - return std::make_shared(std::move(program)); -} - -std::shared_ptr -UnbasedFlatProgram() { - using namespace Libs::Graphics::ShaderRecompiler::IR; - Program program; - program.stage = Libs::Graphics::ShaderType::Compute; - program.srt_plan_complete = true; - program.resource_tracking_complete = true; - AddValueBlock(program); - program.info.uses_dma = true; - return std::make_shared(std::move(program)); -} - -std::shared_ptr -UserDataBufferProgram() { - using namespace Libs::Graphics::ShaderRecompiler::IR; - Program program; - program.stage = Libs::Graphics::ShaderType::Compute; - program.srt_plan_complete = true; - program.resource_tracking_complete = true; - auto &value_block = AddValueBlock(program); - - auto &user_data = value_block.AppendNewInst( - ValueOpcode::GetUserData, {Value(static_cast(0))}); - DescriptorSource source; - source.dwords[0] = Value(&user_data); - source.dwords[1] = Value(0u); - source.dwords[2] = Value(0u); - source.dwords[3] = Value(0u); - source.dword_count = 4; - program.descriptor_sources.push_back(source); - program.info.buffers.push_back({.source = 0}); - return std::make_shared(std::move(program)); -} - -void TestMappedSrtUsesDirectReaderByDefault() { - using namespace Libs::Graphics; - const uint32_t dword = 0x12345678; - auto cached_program = SrtProgram(reinterpret_cast(&dword)); - ShaderStageRuntime stage; - uint32_t specialization_reads = 0; - Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage, - RejectSpecializationRead, - &specialization_reads), - "mapped SRT stage materialization failed"); - Check(specialization_reads == 0, - "ordinary SRT read used the specialization reader"); - Check(stage.program == cached_program && stage.resources != nullptr, - "cache rematerialization did not publish the mapped stage"); - Check(stage.resources->flattened_srt.size() == 1 && - stage.resources->flattened_srt[0] == dword, - "cache rematerialization did not use the direct reader by default"); -} - -void TestUnbasedFlatCacheHitMaterializes() { - using namespace Libs::Graphics; - auto cached_program = UnbasedFlatProgram(); - auto prior_program = std::make_shared(); - auto prior_resources = - std::make_shared(); - ShaderStageRuntime stage{prior_program, prior_resources}; - Check(ShaderMaterializeStageRuntime(cached_program, {}, 0, stage), - "unbased FLAT stage materialization failed"); - Check(stage.program == cached_program && stage.resources != prior_resources, - "unbased FLAT cache hit did not publish its BDA stage snapshot"); -} - -void TestFailedMaterializationPreservesPriorStage() { - using namespace Libs::Graphics; - auto cached_program = UserDataBufferProgram(); - auto prior_program = std::make_shared(); - auto prior_resources = - std::make_shared(); - ShaderStageRuntime stage{prior_program, prior_resources}; - - Check(!ShaderMaterializeStageRuntime(cached_program, {}, 0, stage), - "missing runtime user data did not reject the cached stage"); - Check(stage.program == prior_program && stage.resources == prior_resources, - "failed cache materialization replaced the prior stage"); -} - -} // namespace - -namespace Common { - -int DbgExitHandler(const char *, int, std::string_view) { std::abort(); } - -int DbgExitHandler(const char *, int, fmt::text_style, std::string_view) { - std::abort(); -} - -int DbgExitIfHandler(const char *, const char *, int) { return 1; } - -void DbgExit(int) { std::abort(); } - -} // namespace Common - -int main() { - TestMappedSrtUsesDirectReaderByDefault(); - TestUnbasedFlatCacheHitMaterializes(); - TestFailedMaterializationPreservesPriorStage(); - std::puts("ShaderStageRuntimeTests: all cases passed"); - return 0; -} - -// Keep this focused standalone target self-contained by amalgamating its small -// typed-IR implementation set. -#include "graphics/shader/recompiler/ir/Block.cpp" -#include "graphics/shader/recompiler/ir/Program.cpp" -#include "graphics/shader/recompiler/ir/Type.cpp" -#include "graphics/shader/recompiler/ir/Value.cpp" -#include "graphics/shader/recompiler/ir/opcodes/ValueOpcodes.cpp" diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index e0f5d3b00..e474fc469 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -114,22 +115,52 @@ bool ReadHostTestMemory(void *, uint64_t address, uint32_t *value) { return true; } +struct TestCompileResult { + std::vector spirv; + std::string decoded_dump; + std::string ir_dump; + ShaderRecompiler::IR::Program program; + ShaderRecompiler::IR::ResourceSnapshot resources; +}; + +TestCompileResult RecompileForTest( + std::span code, + const ShaderRecompiler::CompileOptions &options, + ShaderRecompiler::IR::SrtMemoryReader read_memory = nullptr, + void *read_memory_data = nullptr, uint32_t push_data_start_dword = 0) { + auto translated = ShaderRecompiler::TranslateProgram(code, options); + auto plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); + ShaderRecompiler::IR::ResourceSnapshot resources; + ShaderRecompiler::IR::ResourceSpecialization specialization; + const ShaderRecompiler::IR::SrtRuntime runtime{ + .user_data = options.user_data, + .shader_base = reinterpret_cast(code.data()), + .read_memory = read_memory, + .userdata = read_memory_data, + .read_specialization_memory = ReadHostTestMemory, + }; + Check(ShaderRecompiler::IR::MaterializeResources( + plan, runtime, resources, specialization), + "test shader resources did not materialize"); + auto compiled = ShaderRecompiler::CompileProgram( + std::move(translated), options, specialization, push_data_start_dword); + return {std::move(compiled.spirv), std::move(compiled.decoded_dump), + std::move(compiled.ir_dump), std::move(compiled.program), + std::move(resources)}; +} + void CompilePixelRuntime(const ShaderParams ¶ms, ShaderPixelInputInfo &input_info) { auto options = MakeCompileOptions(ShaderType::Pixel); options.shader_hash = params.hash; options.user_data = params.user_data; options.scratch_dwords = input_info.scratch_size_dwords; - options.push_constant_offset = input_info.push_constant_offset; - options.read_specialization_memory = ReadHostTestMemory; options.input_info.pixel = &input_info; - auto result = ShaderRecompiler::Recompile(params.code, options); - input_info.stage.program = - std::make_shared( - std::move(result.program)); - input_info.stage.resources = - std::make_shared( - std::move(result.resources)); + auto result = RecompileForTest(params.code, options); + static std::deque programs; + programs.push_back(std::move(result.program).TakeCompiledInfo()); + input_info.stage.program = &programs.back(); + input_info.stage.resources = std::move(result.resources); } template @@ -1096,7 +1127,17 @@ void TestNativeShaderResourceDependencies() { set_buffer(0, 0x1000, 16, 3); set_buffer(1, 0x1800, 4, 7); set_buffer(2, 0x2000, 0, 64); - const auto writes = CollectShaderBufferWrites(program, resources); + const ShaderRecompiler::IR::CompiledShaderInfo compiled{ + .stage = program.stage, + .shader_hash = program.shader_hash, + .wave_size = program.wave_size, + .user_data_base = program.user_data_base, + .user_data_count = program.user_data_count, + .scratch_dwords = program.scratch_dwords, + .info = program.info, + .bindings = program.bindings, + }; + const auto writes = CollectShaderBufferWrites(compiled, resources); Check(writes == std::vector({{0x1000, 48}, {0x2000, 64}}), "graphics/compute write collector lost or invented a storage-buffer " @@ -1531,7 +1572,7 @@ void TestNewShaderRecompilerSMovB32() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.spirv.empty(), "new shader recompiler produced no SPIR-V"); Check(result.spirv.front() == 0x07230203u, "new shader recompiler did not emit SPIR-V binary"); @@ -1572,7 +1613,7 @@ void TestNewShaderRecompilerSoppMarkers() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_nop 0x00000003"), "new decoder did not decode SOPP s_nop"); Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0x00000000"), @@ -1618,7 +1659,7 @@ void TestNewShaderRecompilerSopkWaitcntMarkers() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 0"), "new decoder did not decode SOPK waitcnt marker"); Check(Common::ContainsStr(result.decoded_dump, "s_waitcnt 65535"), @@ -1647,7 +1688,7 @@ void TestNewShaderRecompilerRdna2ScalarOpcodes() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_bitset1_b32 vcc_lo, 0"), "new decoder did not decode RDNA2 S_BITSET1_B32"); Check(Common::ContainsStr(result.decoded_dump, @@ -1717,7 +1758,7 @@ void TestNewShaderRecompilerScalarVectorAlu() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_add_u32 s2, s0, s1"), "new decoder did not decode SOP2 add"); Check(Common::ContainsStr(result.decoded_dump, "s_addc_u32 s14, s13, 1"), @@ -1819,7 +1860,7 @@ void TestNewShaderRecompilerVop3LaneReadDestinationEncoding() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "v_readfirstlane_b32 s25, v5"), "VOP3 V_READFIRSTLANE_B32 destination was not decoded from VDST"); Check(Common::ContainsStr(result.decoded_dump, "v_readlane_b32 s26, v5, 2"), @@ -2094,7 +2135,7 @@ void TestNewShaderRecompilerMoreAluFamilies() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_movk_i32 s9"), "new decoder did not decode SOPK mov"); Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_u32"), @@ -2939,7 +2980,7 @@ void TestNewShaderRecompilerExpandedAluBatch() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_min_u32"), "new decoder did not decode S_MIN_U32"); Check(Common::ContainsStr(result.decoded_dump, "s_mulk_i32"), @@ -3021,7 +3062,7 @@ void TestNewShaderRecompilerVop3pPackedF16() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "v_pk_add_f16 v114"), "new decoder did not decode old-backed V_PK_ADD_F16"); Check(Common::ContainsStr(result.decoded_dump, "v_pk_mul_f16 v115"), @@ -3145,7 +3186,7 @@ void TestNewShaderRecompilerStagedShaderOps() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_subb_u32 s2, s0, s1"), "new decoder did not decode RDNA2 S_SUBB_U32"); Check(Common::ContainsStr(result.decoded_dump, "s_bitset0_b32 s3, s1"), @@ -3224,7 +3265,7 @@ void TestNewShaderRecompilerBootF16UnaryOpcodes() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "v_sqrt_f16 v4.sdwa(sel=5"), "new decoder did not decode V_SQRT_F16 SDWA high-half destination"); Check(Common::ContainsStr(result.decoded_dump, "v_rndne_f16 v3"), @@ -3293,7 +3334,7 @@ void TestNewShaderRecompilerCapturedVop1SdwaByteConvert() { options.dump_ir = true; options.input_info.pixel = &ps_info; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "V_CVT_F32_UBYTE0 v4, v9.sdwa(sel=4,sext=0)"), "captured V_CVT_F32_UBYTE0 SDWA instruction was not decoded"); @@ -3367,7 +3408,7 @@ void TestNewShaderRecompilerBootB16PackedAndSdwaOpcodes() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "v_lshrrev_b16 v1.sdwa(sel=4"), "new decoder did not decode low-half V_LSHRREV_B16"); Check(Common::ContainsStr(result.decoded_dump, "v_lshlrev_b16 v5.sdwa(sel=5"), @@ -3524,7 +3565,7 @@ void TestNewShaderRecompilerScalarB64Alu() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_mov_b64 s2, s0"), "new decoder did not decode old-backed S_MOV_B64"); Check(Common::ContainsStr(result.decoded_dump, "s_not_b32 s41, s0"), @@ -3699,7 +3740,7 @@ void TestNewShaderRecompilerScalarB64LaneTranslation() { options.user_data = user_data; options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); uint32_t componentwise_u64_values = 0; for (const auto *block : result.program.blocks) { for (const auto &value : *block) { @@ -3738,7 +3779,7 @@ void TestNewShaderRecompilerSignedCompareAlu() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_cmp_gt_i32"), "new decoder did not decode SOPC signed compare"); Check(Common::ContainsStr(result.decoded_dump, "s_cmp_lt_i32"), @@ -3794,7 +3835,7 @@ void TestNewShaderRecompilerSignedMinShiftAlu() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_min_i32"), "new decoder did not decode S_MIN_I32"); Check(Common::ContainsStr(result.decoded_dump, "s_ashr_i32"), @@ -3848,7 +3889,7 @@ void TestNewShaderRecompilerScalarBitfieldAlu() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_cselect_b32"), "new decoder did not decode S_CSELECT_B32"); Check(Common::ContainsStr(result.decoded_dump, "s_abs_i32"), @@ -3922,7 +3963,7 @@ void CheckNewDecoderUnsupported(const uint32_t *shader, uint32_t words, #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - ExpectFatal([&] { (void)ShaderRecompiler::Recompile(code, options); }, + ExpectFatal([&] { (void)RecompileForTest(code, options); }, "unsupported opcode did not terminate shader compilation"); #endif } @@ -3993,7 +4034,7 @@ void TestNewShaderDecoderArchitecture() { const auto completed_options = MakeCompileOptions(ShaderType::Compute); const auto completed_result = - ShaderRecompiler::Recompile(completed_pr_opcodes, completed_options); + RecompileForTest(completed_pr_opcodes, completed_options); Check(Common::ContainsStr(completed_result.decoded_dump, "S_FLBIT_I32_B32 s0, s1") && Common::ContainsStr(completed_result.decoded_dump, @@ -4269,7 +4310,7 @@ void TestNewShaderRecompilerCapturedVopcSdwaCmpxClass() { "captured V_CMPX_CLASS_F32 did not lower to class compare plus EXEC update"); auto options = MakeCompileOptions(ShaderType::Pixel); - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "V_CMPX_CLASS_F32 exec_lo, v13, vcc_lo"), "captured SDWA V_CMPX_CLASS_F32 was not present in the decoded dump"); @@ -4401,9 +4442,8 @@ void TestNewShaderRecompilerMemoryFamilyTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; options.user_data = user_data; - options.read_memory = ReadZeroTestMemory; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options, ReadZeroTestMemory); Check(Common::ContainsStr(result.decoded_dump, "s_load_dword"), "new decoder did not decode SMEM dword load"); Check(Common::ContainsStr(result.decoded_dump, "s_buffer_load_dword"), @@ -4496,7 +4536,7 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { auto raw_options = MakeCompileOptions(ShaderType::Pixel); raw_options.user_data = raw_user_data; - auto raw = ShaderRecompiler::Recompile(raw_shader, raw_options); + auto raw = RecompileForTest(raw_shader, raw_options); const auto *address_binding = ShaderRecompiler::IR::FindBinding( raw.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::BdaPagetable); @@ -4540,7 +4580,7 @@ void TestNewShaderRecompilerScalarMemoryBindingDomains() { auto buffer_options = MakeCompileOptions(ShaderType::Pixel); buffer_options.user_data = buffer_user_data; - auto buffer = ShaderRecompiler::Recompile(buffer_shader, buffer_options); + auto buffer = RecompileForTest(buffer_shader, buffer_options); const auto *buffer_binding = ShaderRecompiler::IR::FindBinding( buffer.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Buffers); @@ -4578,7 +4618,7 @@ void TestNewShaderRecompilerImageQueryTranslation() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_get_lod"), "new decoder did not decode MIMG image get-lod query"); Check(Common::ContainsStr(result.decoded_dump, "dmask=0x3"), @@ -4622,7 +4662,7 @@ void TestNewShaderRecompilerCubeSampleCoordinates() { auto options = MakeCompileOptions(ShaderType::Compute); options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(result.program.info.images.size() == 1 && result.program.info.images[0].cube, "cube descriptor identity was not preserved through compilation"); @@ -4668,7 +4708,7 @@ void TestNewShaderRecompilerImageSampleVariants() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_sample_l"), "new decoder did not decode IMAGE_SAMPLE_L through shared MIMG path"); Check(Common::ContainsStr(result.decoded_dump, "image_sample_b"), @@ -4756,7 +4796,7 @@ void TestNewShaderRecompilerImageSampleA16SamplerCoords() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d sample_flags=a16 addr_components=3"), "3D IMAGE_SAMPLE with MIMG A16 bit did not decode as three A16 sampler " @@ -4798,7 +4838,7 @@ void TestNewShaderRecompilerImageSampleOpcodeAliases() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_sample_a"), "MIMG opcode 0xa0 should decode as image_sample_a alias"); Check(Common::ContainsStr(result.decoded_dump, "image_sample_b_a"), @@ -4834,7 +4874,7 @@ void TestNewShaderRecompilerImageSampleA16ExceptionComponents() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); return result; }; @@ -4880,7 +4920,7 @@ void TestNewShaderRecompilerImageLoadA16UintCoords() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_load"), "A16 IMAGE_LOAD did not decode"); Check(Common::ContainsStr(result.ir_dump, "ImageLoad v20"), @@ -4924,7 +4964,7 @@ void TestNewShaderRecompilerPixelImageSampleLodSelection() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); return result; }; @@ -5035,7 +5075,7 @@ void TestNewShaderRecompilerImageViewDimensions() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_array"), "MIMG DIM did not decode 2D-array image view"); Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), @@ -5111,7 +5151,7 @@ void TestNewShaderRecompilerStorageImage1DDescriptorVariants() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "image_dim=1d"), "1D storage dimension did not survive into IR metadata"); Check(Common::ContainsStr(result.ir_dump, "image_dim=1d_array"), @@ -5141,7 +5181,7 @@ void TestNewShaderRecompilerNullImageUsesCanonical2DView() { auto options = MakeCompileOptions(ShaderType::Compute); options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); Check(SpirvSourceHasInstructionUsing(source, "OpAccessChain", "image_3 "), @@ -5172,7 +5212,7 @@ void TestNewShaderRecompilerImageGatherVariants() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz"), "new decoder did not decode IMAGE_GATHER4_LZ"); Check(Common::ContainsStr(result.decoded_dump, "image_gather4_lz_o"), @@ -5258,7 +5298,7 @@ void TestNewShaderRecompilerImageLoadVariants() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_load"), "new decoder did not decode MIMG image load"); Check(Common::ContainsStr(result.decoded_dump, "image_load_mip"), @@ -5304,7 +5344,7 @@ void TestNewShaderRecompilerImageLoad2DMsaa() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_dim=2d_msaa") && Common::ContainsStr(result.ir_dump, "image_dim=2d_msaa") && Common::ContainsStr(result.ir_dump, "image_addr=3 image_mip=0"), @@ -5345,7 +5385,7 @@ void TestNewShaderRecompilerImageStoreTranslation() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_store"), "new decoder did not decode MIMG image store"); Check(Common::ContainsStr(result.decoded_dump, "image_store_mip"), @@ -5396,7 +5436,7 @@ void TestNewShaderRecompilerStorageImage3DDescriptorVariant() { options.input_info.compute = &input_info; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), "MIMG store did not decode the RDNA2 3D instruction dimension"); const auto has_3d_store = std::ranges::any_of( @@ -5450,7 +5490,7 @@ void TestNewShaderRecompilerStorageImage2DDescriptorOverridesMimg3D() { options.input_info.compute = &input_info; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_dim=3d"), "test MIMG store should decode as a 3D instruction"); Check(SpirvContainsTypeImage(result.spirv, SpirvDim2D, 0, 2), @@ -5491,7 +5531,7 @@ void TestNewShaderRecompilerImageAtomicTranslation() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "image_atomic_add"), "new decoder did not decode MIMG image atomic add"); Check(Common::ContainsStr(result.decoded_dump, "image_atomic_umin"), @@ -5555,7 +5595,7 @@ void TestNewShaderRecompilerVintrpTranslation() { options.input_info.pixel = &ps_info; options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "v_interp_p1_f32"), "new decoder did not decode VINTRP P1"); Check(Common::ContainsStr(result.decoded_dump, "v_interp_p2_f32"), @@ -5594,7 +5634,7 @@ void TestNewShaderRecompilerVintrpTranslation() { options.input_info.pixel = &remapped_ps_info; - auto remapped_result = ShaderRecompiler::Recompile(remapped_shader, options); + auto remapped_result = RecompileForTest(remapped_shader, options); Check( Common::ContainsStr(remapped_result.ir_dump, "input_attr=2 input_chan=0"), "remapped VINTRP did not preserve raw pixel attribute metadata"); @@ -5618,7 +5658,7 @@ void TestNewShaderRecompilerVintrpTranslation() { options.input_info.pixel = &duplicate_ps_info; auto duplicate_result = - ShaderRecompiler::Recompile(duplicate_location_shader, options); + RecompileForTest(duplicate_location_shader, options); Check(ProgramInputCount(duplicate_result.program, ShaderRecompiler::IR::StageInputKind::Parameter) == 2, "duplicate-location VINTRP shader did not reflect both raw parameter " @@ -5640,7 +5680,7 @@ void TestNewShaderRecompilerVintrpTranslation() { options.input_info.pixel = &flat_ps_info; - auto flat_result = ShaderRecompiler::Recompile(flat_shader, options); + auto flat_result = RecompileForTest(flat_shader, options); Check(SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 14u), "flat VINTRP input did not emit a Flat decoration"); Check(!SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 13u), @@ -5657,7 +5697,7 @@ void TestNewShaderRecompilerVintrpTranslation() { options.input_info.pixel = &no_perspective_ps_info; auto no_perspective_result = - ShaderRecompiler::Recompile(flat_shader, options); + RecompileForTest(flat_shader, options); Check(SpirvHasDecorationValueWithDecoration(no_perspective_result.spirv, 30u, 0u, 13u), "no-perspective VINTRP input did not emit a NoPerspective decoration"); @@ -5686,7 +5726,7 @@ void TestCustomVintrpMovTranslation() { options.input_info.pixel = &custom_ps_info; options.dump_ir = true; - auto custom_result = ShaderRecompiler::Recompile(shader, options); + auto custom_result = RecompileForTest(shader, options); Check(Common::ContainsStr( custom_result.ir_dump, "GetInterpolationParameter 0x00000000, 0x00000003, 0x00000000"), @@ -5732,7 +5772,7 @@ void TestCustomVintrpMovTranslation() { custom_ps_info.custom_interpolation_mask = 0; custom_ps_info.interpolator_settings[0] = 0; - auto standard_result = ShaderRecompiler::Recompile(shader, options); + auto standard_result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(standard_result.spirv, 131u) == 2u, "standard VINTRP P10/P20 did not subtract P0 exactly once each"); CheckSpirvBinaryValidates(standard_result.spirv); @@ -5747,7 +5787,7 @@ void TestCustomVintrpMovTranslation() { flat_ps_info.input_num = 1; flat_ps_info.interpolator_settings[0] = 0x00000400u; options.input_info.pixel = &flat_ps_info; - auto flat_result = ShaderRecompiler::Recompile(flat_shader, options); + auto flat_result = RecompileForTest(flat_shader, options); Check(!SpirvContainsCapability(flat_result.spirv, 5284u), "flat P0 unexpectedly required fragment barycentric support"); Check(SpirvHasDecorationValueWithDecoration(flat_result.spirv, 30u, 0u, 14u), @@ -5766,14 +5806,14 @@ void TestCustomVintrpMovTranslation() { ShaderPixelInputInfo mixed_ps_info{}; mixed_ps_info.input_num = 1; options.input_info.pixel = &mixed_ps_info; - auto mixed_result = ShaderRecompiler::Recompile(mixed_shader, options); + auto mixed_result = RecompileForTest(mixed_shader, options); Check(SpirvHasDecorationValue(mixed_result.spirv, 11u, 5286u) && SpirvInstructionOpcodeCount(mixed_result.spirv, 133u) == 3u, "mixed ordinary/P0 input was not interpolated from per-vertex values"); CheckSpirvBinaryValidates(mixed_result.spirv); mixed_ps_info.ps_no_perspective = true; - auto mixed_linear_result = ShaderRecompiler::Recompile(mixed_shader, options); + auto mixed_linear_result = RecompileForTest(mixed_shader, options); Check(SpirvHasDecorationValue(mixed_linear_result.spirv, 11u, 5287u), "mixed linear input did not use BaryCoordNoPerspKHR"); CheckSpirvBinaryValidates(mixed_linear_result.spirv); @@ -5977,7 +6017,7 @@ void TestNewShaderRecompilerNativeWideBufferIr() { }; auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); std::array counts{}; for (const auto *block : result.program.blocks) { for (const auto &inst : *block) { @@ -6024,7 +6064,7 @@ void TestNewShaderRecompilerBufferSignedLoadTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sbyte"), "new decoder did not decode buffer signed byte load"); Check(Common::ContainsStr(result.decoded_dump, "buffer_load_sshort"), @@ -6061,7 +6101,7 @@ void TestNewShaderRecompilerBufferSubDwordStoreTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "buffer_store_byte"), "new decoder did not decode buffer byte store"); Check(Common::ContainsStr(result.decoded_dump, "buffer_store_short"), @@ -6113,7 +6153,7 @@ void TestNewShaderRecompilerMubufFormatTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "BUFFER_LOAD_FORMAT_X"), "new decoder did not decode MUBUF format-x load"); Check(Common::ContainsStr(result.decoded_dump, "BUFFER_LOAD_FORMAT_XYZW"), @@ -6154,7 +6194,7 @@ void TestNewShaderRecompilerFormattedStoreUsesRuntimeArrayLengthOnly() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(result.resources.buffers.size() == 1 && result.resources.buffers[0].dwords[2] == 5u, "formatted store test did not preserve descriptor NumRecords"); @@ -6194,7 +6234,7 @@ void TestNewShaderRecompilerTypedBufferTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_LOAD_FORMAT_X"), "new decoder did not decode typed buffer format-x load"); Check(Common::ContainsStr(result.decoded_dump, "TBUFFER_LOAD_FORMAT_XY"), @@ -6240,7 +6280,7 @@ void TestNewShaderRecompilerFlatOldBackedTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "flat_load_ubyte"), "new decoder did not decode old-backed FLAT ubyte load"); Check(Common::ContainsStr(result.ir_dump, "FlatLoadUbyte v9"), @@ -6269,7 +6309,7 @@ void TestNewShaderRecompilerUnbasedFlatUsesBda() { auto options = MakeCompileOptions(ShaderType::Pixel); - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(result.program.info.uses_dma && ShaderRecompiler::IR::FindBinding( result.program.bindings, @@ -6297,7 +6337,7 @@ void TestNewShaderRecompilerFlatSignedLoadTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "flat_load_sbyte"), "new decoder did not decode flat signed byte load"); Check(Common::ContainsStr(result.decoded_dump, "flat_load_sshort"), @@ -6348,7 +6388,7 @@ void TestNewShaderRecompilerFlatStoreTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "flat_store_byte"), "new decoder did not decode flat byte store"); Check(Common::ContainsStr(result.decoded_dump, "flat_store_short"), @@ -6467,7 +6507,7 @@ void TestNewShaderRecompilerAtomicTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_swap"), "new decoder did not decode buffer atomic swap"); Check(Common::ContainsStr(result.decoded_dump, "buffer_atomic_add"), @@ -6623,7 +6663,7 @@ void TestNewShaderRecompilerDsReadWrite2Translation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE2_B32"), "new decoder did not decode old-backed DS write2"); Check(Common::ContainsStr(result.decoded_dump, "DS_READ2_B32"), @@ -6674,7 +6714,7 @@ void TestNewShaderRecompilerDsSubDwordTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "ds_write_b8"), "new decoder did not decode DS byte write"); Check(Common::ContainsStr(result.decoded_dump, "ds_write_b16"), @@ -6778,7 +6818,7 @@ void TestNewShaderRecompilerDsWideAndAtomicTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B64"), "new decoder did not decode DS b64 write"); Check(Common::ContainsStr(result.decoded_dump, "DS_WRITE_B96"), @@ -6824,7 +6864,7 @@ void TestNewShaderRecompilerDsSwizzleTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "ds_swizzle_b32"), "new decoder did not decode DS swizzle"); Check(Common::ContainsStr(result.decoded_dump, "offset=31"), @@ -6874,7 +6914,7 @@ void TestNewShaderRecompilerDsAddtidTranslation() { options.dump_ir = true; options.input_info.compute = &input_info; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "ds_write_addtid_b32"), "new decoder did not decode DS write addtid"); Check(Common::ContainsStr(result.decoded_dump, "ds_read_addtid_b32"), @@ -6912,7 +6952,7 @@ void TestNewShaderRecompilerDsFloatMinMaxTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "ds_min_f32"), "new decoder did not decode DS float min"); Check(Common::ContainsStr(result.decoded_dump, "ds_max_f32"), @@ -6947,7 +6987,7 @@ void TestNewShaderRecompilerCfgStraightLine() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "CFG:"), "CFG dump was not emitted"); Check(Common::ContainsStr(result.ir_dump, "block_0"), @@ -6970,7 +7010,7 @@ void TestNewShaderRecompilerCfgIfElse() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "condition=scc1"), "if/else branch condition missing"); Check(SpirvContainsOpcode(result.spirv, 247), @@ -6998,7 +7038,7 @@ void TestNewShaderRecompilerCfgConsecutiveNativePhis() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(CountSourceOccurrences(result.ir_dump, " = Phi") == 2u && SpirvInstructionOpcodeCount(result.spirv, 245u) == 2u, "consecutive typed Phis were not emitted as two native OpPhi " @@ -7016,7 +7056,7 @@ void TestNewShaderRecompilerStructuredU64Phi() { EncodeSopp(0x01), }; auto options = MakeCompileOptions(ShaderType::Compute); - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback, "U64 Phi fixture did not select structured mode"); @@ -7040,8 +7080,7 @@ void TestNewShaderRecompilerStructuredU64Phi() { {IR::Value(&phi), IR::Value(1u)}); ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); - auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, result.resources, - options.input_info); + auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto before = MeasureSpirv(result.spirv); const auto after = MeasureSpirv(spirv); @@ -7060,7 +7099,7 @@ void TestNewShaderRecompilerCfgTerminalExitMergePS() { auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "terminal PS branch should stay on structured path"); Check(!Common::ContainsStr(result.ir_dump, @@ -7083,7 +7122,7 @@ void TestNewShaderRecompilerCfgPostEndTargetMergePS() { auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "0x0000000c: S_MOV_B32"), "post-end branch target was not decoded"); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), @@ -7111,7 +7150,7 @@ void TestNewShaderRecompilerCfgLoopBreakContinue() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "backedge"), "loop backedge was not detected"); Check(Common::ContainsStr(result.ir_dump, "loop_header=1"), @@ -7140,7 +7179,7 @@ void TestNewShaderRecompilerCfgLoopHeaderDynamicScalarBufferLoadStructured() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - ExpectFatal([&] { (void)ShaderRecompiler::Recompile(shader, options); }, + ExpectFatal([&] { (void)RecompileForTest(shader, options); }, "self-modifying scalar-buffer descriptor did not terminate " "compilation"); } @@ -7160,7 +7199,7 @@ void TestNewShaderRecompilerCfgLoopHeaderBufferLoadDispatcher() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - ExpectFatal([&] { (void)ShaderRecompiler::Recompile(shader, options); }, + ExpectFatal([&] { (void)RecompileForTest(shader, options); }, "self-modifying vector-buffer descriptor did not terminate " "compilation"); } @@ -7201,7 +7240,7 @@ void TestNewShaderRecompilerCfgLoopHeaderDsAppendConsumeStructured() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "DS append/consume loop did not stay structured"); Check(!result.program.dispatcher_fallback, @@ -7226,7 +7265,7 @@ void TestNewShaderRecompilerCfgLoopHeaderDsReadStructured() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured") && !result.program.dispatcher_fallback, "DS read loop header did not stay structured"); @@ -7255,7 +7294,7 @@ void TestNewShaderRecompilerCfgLoopHeaderDsRead2B64Structured() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured") && !result.program.dispatcher_fallback, "DS read2 b64 loop header did not stay structured"); @@ -7282,7 +7321,7 @@ void TestNewShaderRecompilerCfgSharedOuterAndLoopMerge() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "shared outer/loop merge should stay on structured path"); Check( @@ -7311,7 +7350,7 @@ void TestNewShaderRecompilerCfgLoopEarlyBreakNoSelection() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "loop early-break CFG did not stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, @@ -7342,7 +7381,7 @@ void TestNewShaderRecompilerCfgNestedLoopNonlocalExitDispatcher() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), "nested-loop nonlocal exit did not select dispatcher fallback"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, @@ -7366,7 +7405,7 @@ void TestNewShaderRecompilerCfgNestedLoopLocalExitNoSelection() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "nested local loop exit did not stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) >= 2, @@ -7446,7 +7485,7 @@ void TestNewShaderRecompilerCfgMixedContinueNonmergeExitDispatcher() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), "mixed continue/nonmerge exit did not select dispatcher fallback"); Check(SpirvInstructionOpcodeCount(result.spirv, 251) != 0, @@ -7466,7 +7505,7 @@ void TestNewShaderRecompilerCfgConditionalLatchNoSelection() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "conditional latch did not stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, @@ -7489,7 +7528,7 @@ void TestNewShaderRecompilerCfgDirectConditionalLatchNoSelection() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "direct conditional latch did not stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, @@ -7521,7 +7560,7 @@ void TestNewShaderRecompilerCfgLoopEarlyContinuesNoSelection() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "loop early continues should stay on structured path"); Check(SpirvInstructionOpcodeCount(result.spirv, 246) != 0, @@ -7562,7 +7601,7 @@ void TestNewShaderRecompilerCfgLoopGatewaySelection() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured") && !result.program.dispatcher_fallback, "loop-control gateway selection unexpectedly selected dispatcher"); @@ -7617,7 +7656,7 @@ void TestNewShaderRecompilerCfgConditionalLoopHeaderSelection() { "guest conditional was not separated from the loop header"); auto options = MakeCompileOptions(ShaderType::Compute); - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, "conditional loop-header SPIR-V has the wrong loop-merge count"); Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 1u, @@ -7661,7 +7700,7 @@ void TestNewShaderRecompilerCfgMultipleLoopLatches() { "canonical continue does not join both native latches"); auto options = MakeCompileOptions(ShaderType::Compute); - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(result.spirv, 246) == 1u, "multiple-latch SPIR-V has the wrong loop-merge count"); Check(SpirvInstructionOpcodeCount(result.spirv, 247) == 0u, @@ -7683,7 +7722,7 @@ void TestNewShaderRecompilerCfgDuplicateMergeStructuredSplit() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=structured"), "duplicate merge CFG did not stay on structured path"); Check( @@ -7753,7 +7792,7 @@ void TestNewShaderRecompilerCfgNestedEarlyExitLoopForwarders() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured"), "nested early-exit loop unexpectedly selected dispatcher fallback"); @@ -7835,7 +7874,7 @@ void TestNewShaderRecompilerCfgExecSccSharedArm() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured"), "EXEC/SCC shared-arm epilogue did not stay structured"); @@ -7896,7 +7935,7 @@ void TestNewShaderRecompilerCfgNestedTailEarlyExit() { auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured") && SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, @@ -7953,7 +7992,7 @@ void TestNewShaderRecompilerCfgRoutesInnerSharedExitFirst() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured") && SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, @@ -8014,7 +8053,7 @@ void TestNewShaderRecompilerCfgLoopSharedRegion() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured") && SpirvInstructionOpcodeCount(result.spirv, 246) == 1u && @@ -8084,7 +8123,7 @@ void TestNewShaderRecompilerCfgOverlappingEarlyExitLadder() { auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured"), "overlapping early-exit ladder did not stay structured"); @@ -8132,7 +8171,7 @@ void TestNewShaderRecompilerCfgNestedEarlyExitSharedTerminal() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured") && SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, @@ -8171,7 +8210,7 @@ void TestNewShaderRecompilerCfgSharedTerminalEarlyExit() { auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured"), "shared-terminal early exit selected dispatcher control flow"); @@ -8206,7 +8245,7 @@ void TestNewShaderRecompilerCfgPrunesUnreachableSelectionEntry() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!result.program.dispatcher_fallback && Common::ContainsStr(result.ir_dump, "mode=structured") && SpirvInstructionOpcodeCount(result.spirv, 251) == 0u, @@ -8226,7 +8265,7 @@ void TestNewShaderRecompilerCfgIrreducibleDispatcher() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "irreducible CFG"), "irreducible CFG reason was not retained"); Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), @@ -8249,7 +8288,7 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { EncodeSopp(0x01), }; auto options = MakeCompileOptions(ShaderType::Compute); - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(result.program.dispatcher_fallback && result.program.blocks.size() >= 3u, "U32x3 spill fixture did not select dispatcher mode"); @@ -8263,8 +8302,8 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { IR::IREmitter use(prologue_program.blocks[1]); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); ShaderRecompiler::Spirv::AnalyzeProgramRequirements(prologue_program); - auto spirv = ShaderRecompiler::Spirv::EmitProgram( - prologue_program, result.resources, options.input_info); + auto spirv = ShaderRecompiler::Spirv::EmitProgram(prologue_program, + options.input_info); CheckSpirvBinaryValidates(spirv); const auto after = MeasureSpirv(spirv); Check(after.function_variables == before.function_variables && @@ -8272,7 +8311,7 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { "dispatcher spilled a value defined by its dominating prologue"); } - auto spill_result = ShaderRecompiler::Recompile(shader, options); + auto spill_result = RecompileForTest(shader, options); auto program = std::move(spill_result.program); IR::IREmitter definition(program.blocks[1]); const auto vector = @@ -8283,8 +8322,8 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() { use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(1u)}); ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); - auto spirv = ShaderRecompiler::Spirv::EmitProgram( - program, spill_result.resources, options.input_info); + auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, + options.input_info); CheckSpirvBinaryValidates(spirv); const auto after = MeasureSpirv(spirv); Check(after.function_variables == before.function_variables + 1u && @@ -8298,7 +8337,7 @@ void TestNewShaderRecompilerU64PairTranslation() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - CompileResult result; + TestCompileResult result; const uint32_t lshrrev_b64_shader[] = { 0xd7000001u, 0x0000d50cu, // v_lshrrev_b64 v[1:2], v12, vcc @@ -8315,7 +8354,7 @@ void TestNewShaderRecompilerU64PairTranslation() { decoded_lshrrev_b64.src0.reg == 12u && decoded_lshrrev_b64.src1.kind == Decoder::OperandKind::VccLo, "decoder rejected captured VOP3 V_LSHRREV_B64 fields"); - result = ShaderRecompiler::Recompile(lshrrev_b64_shader, options); + result = RecompileForTest(lshrrev_b64_shader, options); Check(Common::ContainsStr(result.decoded_dump, "V_LSHRREV_B64 v1, v12, vcc_lo"), "captured VOP3 V_LSHRREV_B64 was not present in the decoded dump"); @@ -8338,7 +8377,7 @@ void TestNewShaderRecompilerU64PairTranslation() { Decoder::OperandKind::IntegerInlineConstant && decoded_lshlrev_b64.src1.value == 1u, "decoder rejected captured VOP3 V_LSHLREV_B64 fields"); - result = ShaderRecompiler::Recompile(lshlrev_b64_shader, options); + result = RecompileForTest(lshlrev_b64_shader, options); Check(Common::ContainsStr(result.decoded_dump, "V_LSHLREV_B64 v33, v3, 1"), "captured VOP3 V_LSHLREV_B64 was not present in the decoded dump"); @@ -8357,7 +8396,7 @@ void TestNewShaderRecompilerU64PairTranslation() { decoded_cmpx_i64.src1.kind == Decoder::OperandKind::Vgpr && decoded_cmpx_i64.src1.reg == 7u, "decoder rejected captured VOP3 V_CMPX_NE_I64 fields"); - result = ShaderRecompiler::Recompile(cmpx_i64_shader, options); + result = RecompileForTest(cmpx_i64_shader, options); Check(Common::ContainsStr(result.decoded_dump, "V_CMPX_NE_I64 exec_lo, 0, v7"), "captured VOP3 V_CMPX_NE_I64 was not present in the decoded dump"); @@ -8375,7 +8414,7 @@ void TestNewShaderRecompilerU64PairTranslation() { decoded_cmpx.src1.kind == Decoder::OperandKind::Vgpr && decoded_cmpx.src1.reg == 7u, "decoder rejected captured V_CMPX_NE_U64 fields"); - result = ShaderRecompiler::Recompile(cmpx_shader, options); + result = RecompileForTest(cmpx_shader, options); CheckSpirvBinaryValidates(result.spirv); const uint32_t cmpx_v2_shader[] = { @@ -8391,14 +8430,14 @@ void TestNewShaderRecompilerU64PairTranslation() { decoded_cmpx_v2.src1.kind == Decoder::OperandKind::Vgpr && decoded_cmpx_v2.src1.reg == 2u, "decoder rejected reported VOPC V_CMPX_NE_U64 fields"); - result = ShaderRecompiler::Recompile(cmpx_v2_shader, options); + result = RecompileForTest(cmpx_v2_shader, options); Check(Common::ContainsStr(result.decoded_dump, "V_CMPX_NE_U64 exec_lo, 0, v2"), "reported VOPC V_CMPX_NE_U64 was not present in the decoded dump"); CheckSpirvBinaryValidates(result.spirv); const uint32_t empty_shader[] = {EncodeSopp(0x01)}; - result = ShaderRecompiler::Recompile(empty_shader, options); + result = RecompileForTest(empty_shader, options); auto program = std::move(result.program); IR::IREmitter ir(program.blocks.front()); @@ -8425,8 +8464,7 @@ void TestNewShaderRecompilerU64PairTranslation() { } ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); - auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, result.resources, - options.input_info); + auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto source = DisassembleSpirvBinary(spirv); Check(!Common::ContainsStr(source, "OpCapability Int64") && @@ -8448,7 +8486,7 @@ void TestNewShaderRecompilerU64PairTranslation() { EncodeSopp(0x05, 2), EncodeSopp(0x02, 0), EncodeSopp(0x05, 0xfffeu), EncodeSopp(0x02, 0xfffeu), EncodeSopp(0x01), }; - result = ShaderRecompiler::Recompile(dispatcher_shader, options); + result = RecompileForTest(dispatcher_shader, options); Check(result.program.dispatcher_fallback && result.program.blocks.size() >= 3u, "U64 spill fixture did not select dispatcher mode"); @@ -8460,8 +8498,7 @@ void TestNewShaderRecompilerU64PairTranslation() { use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)}); ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); spirv.clear(); - spirv = ShaderRecompiler::Spirv::EmitProgram(program, result.resources, - options.input_info); + spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); CheckSpirvBinaryValidates(spirv); const auto before = MeasureSpirv(result.spirv); const auto after = MeasureSpirv(spirv); @@ -8491,7 +8528,7 @@ void TestNewShaderRecompilerBufferLoadsGuardedByExec() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "BUFFER_LOAD_DWORD"), "buffer load guard regression did not decode MUBUF load"); CheckSpirvBinaryValidates(result.spirv); @@ -8529,7 +8566,7 @@ void TestNewShaderRecompilerBufferAtomicsGuardedByBounds() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "BUFFER_ATOMIC_ADD"), "buffer atomic bounds regression did not decode MUBUF atomic"); CheckSpirvBinaryValidates(result.spirv); @@ -8577,7 +8614,7 @@ void TestNewShaderRecompilerBranchConditionForms() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, c.condition), "branch condition was not preserved"); Check(SpirvContainsOpcode(result.spirv, 250), @@ -8602,7 +8639,7 @@ void TestNewShaderRecompilerSetpcBranch() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "s_setpc_b64"), "S_SETPC_B64 was not decoded"); Check(Common::ContainsStr(result.ir_dump, "successors=["), @@ -8638,7 +8675,7 @@ void TestNewShaderRecompilerSetpcJumpTable() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), "S_SETPC_B64 jump table did not select dispatcher fallback"); const auto jump = std::find_if( @@ -8742,7 +8779,7 @@ void TestNewShaderRecompilerSetpcDwordJumpTable() { auto options = MakeCompileOptions(ShaderType::Compute); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.ir_dump, "mode=dispatcher"), "subtractive S_SETPC_B64 table did not select dispatcher fallback"); const auto jump = std::find_if( @@ -8790,7 +8827,7 @@ void TestNewShaderRecompilerExpVertexOutputs() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(Common::ContainsStr(result.decoded_dump, "target=0x0c"), "POS export was not decoded"); Check(Common::ContainsStr(result.decoded_dump, "target=0x20"), @@ -8815,7 +8852,7 @@ void TestNewShaderRecompilerClipDisabledPosition() { const auto compile = [&](const ShaderVertexInputInfo &vertex) { auto options = MakeCompileOptions(ShaderType::Vertex); options.input_info.vertex = &vertex; - return ShaderRecompiler::Recompile(shader, options); + return RecompileForTest(shader, options); }; ShaderVertexInputInfo regular{}; @@ -8845,6 +8882,19 @@ void TestNewShaderRecompilerClipDisabledPosition() { shifted.clip_space.offset[0] += 1.0f; Check(MakeStageStaticKey(clipped) != MakeStageStaticKey(shifted), "clip-disabled viewport transform is absent from the shader cache key"); + + ShaderVertexInputInfo layout_a{}; + layout_a.resources_num = 1; + layout_a.buffers_num = 1; + layout_a.buffers[0].attr_num = 1; + layout_a.buffers[0].attr_indices[0] = 0; + layout_a.buffers[0].stride = 16; + auto layout_b = layout_a; + layout_b.buffers[0].stride = 32; + layout_b.buffers[0].fetch_index = 1; + layout_b.buffers[0].attr_offsets[0] = 4; + Check(MakeStageStaticKey(layout_a) == MakeStageStaticKey(layout_b), + "pipeline-only vertex layout fragmented the shader module cache key"); } void TestNewShaderRecompilerAuxPositionExports() { @@ -8854,7 +8904,7 @@ void TestNewShaderRecompilerAuxPositionExports() { auto options = MakeCompileOptions(ShaderType::Vertex); options.input_info.vertex = &vertex; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); return result; }; @@ -8944,7 +8994,7 @@ void TestNewShaderRecompilerAuxPositionExports() { vertex.pa_cl_vs_out_cntl = control; auto options = MakeCompileOptions(ShaderType::Vertex); options.input_info.vertex = &vertex; - ExpectFatal([&] { (void)ShaderRecompiler::Recompile(shader, options); }, + ExpectFatal([&] { (void)RecompileForTest(shader, options); }, message); }; rejects(0x00280000u, 0x4u, @@ -9008,7 +9058,7 @@ void TestDemandDrivenSpirvDeclarations() { SetImageTestFormat(&user_data, 0, format); auto options = MakeCompileOptions(ShaderType::Compute); options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); return result.spirv; }; @@ -9021,7 +9071,7 @@ void TestDemandDrivenSpirvDeclarations() { auto query_options = MakeCompileOptions(ShaderType::Pixel); query_options.input_info.pixel = &query_pixel_info; query_options.user_data = query_user_data; - auto query_result = ShaderRecompiler::Recompile(query, query_options); + auto query_result = RecompileForTest(query, query_options); CheckSpirvBinaryValidates(query_result.spirv); const auto &query_spirv = query_result.spirv; Check(SpirvInstructionOpcodeCount(query_spirv, 105u) == 1u, @@ -9067,7 +9117,7 @@ void TestDemandDrivenSpirvDeclarations() { auto glsl_pixel_info = RegressionPixelInputInfo(); auto glsl_options = MakeCompileOptions(ShaderType::Pixel); glsl_options.input_info.pixel = &glsl_pixel_info; - auto glsl_result = ShaderRecompiler::Recompile(glsl, glsl_options); + auto glsl_result = RecompileForTest(glsl, glsl_options); CheckSpirvBinaryValidates(glsl_result.spirv); Check(MeasureSpirv(glsl_result.spirv).ext_inst_imports == 1u, "multiple GLSL operations emitted duplicate imports"); @@ -9812,7 +9862,7 @@ void TestNewShaderRecompilerZeroInitialRegisterState() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(!Common::ContainsStr(result.ir_dump, "UndefU32"), "SSA initial state left guest registers undefined"); const auto source = DisassembleSpirvBinary(result.spirv); @@ -9830,7 +9880,7 @@ void TestNewShaderRecompilerZeroInitialRegisterState() { EncodeSopp(0x01), }; auto explicit_zero = - ShaderRecompiler::Recompile(explicit_zero_shader, options); + RecompileForTest(explicit_zero_shader, options); Check(result.spirv == explicit_zero.spirv, "implicit and explicit zero register state emitted different SPIR-V"); } @@ -9851,7 +9901,7 @@ void TestNewShaderRecompilerVertexSystemInputsWithoutMirrors() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(ProgramHasInput(result.program, StageInputKind::VertexIndex), "vertex shader missing VertexIndex input"); Check(ProgramHasInput(result.program, StageInputKind::InstanceIndex), @@ -9879,7 +9929,7 @@ void TestNewShaderRecompilerVertexExportUsesInvocationExecMask() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); Check(!Common::ContainsStr(source, "OpLoad %uint %gl_SubgroupInvocationID"), @@ -9904,7 +9954,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(local_shader, options); + auto result = RecompileForTest(local_shader, options); CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); Check( @@ -9922,7 +9972,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 2, 3), // POS0 EncodeSopp(0x01), }; - result = ShaderRecompiler::Recompile(wqm_shader, options); + result = RecompileForTest(wqm_shader, options); CheckSpirvBinaryValidates(result.spirv); const auto wqm_source = DisassembleSpirvBinary(result.spirv); Check(Common::ContainsStr(wqm_source, "OpCapability GroupNonUniformBallot") && @@ -9933,7 +9983,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { auto wave32_options = options; wave32_options.wave_size = 32u; - result = ShaderRecompiler::Recompile(wqm_shader, wave32_options); + result = RecompileForTest(wqm_shader, wave32_options); CheckSpirvBinaryValidates(result.spirv); Check(SpirvInstructionOpcodeCount(result.spirv, 132u) == 1u, "wave32 WQM retained the unused high ballot-word expansion"); @@ -9946,7 +9996,7 @@ void TestNewShaderRecompilerPerInvocationMasksWithoutMirrors() { EncodeExp1(0, 1, 2, 3), // POS0 EncodeSopp(0x01), }; - result = ShaderRecompiler::Recompile(cross_lane_shader, options); + result = RecompileForTest(cross_lane_shader, options); CheckSpirvBinaryValidates(result.spirv); Check(Common::ContainsStr(DisassembleSpirvBinary(result.spirv), "OpGroupNonUniformBallot"), @@ -9968,7 +10018,7 @@ void TestNewShaderRecompilerPerInvocationU64Complement() { auto options = MakeCompileOptions(ShaderType::Vertex); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); const auto source = DisassembleSpirvBinary(result.spirv); Check(Common::ContainsStr(source, "OpLogicalNot"), @@ -9993,7 +10043,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { auto options = MakeCompileOptions(ShaderType::Pixel); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(SpirvContainsOpcode(result.spirv, 62), "pixel export SPIR-V lacks OpStore"); Check(SpirvContainsExtInst(result.spirv, 62), @@ -10005,7 +10055,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { ShaderPixelInputInfo uint16_info; uint16_info.target_output_mode[0] = 7; options.input_info.pixel = &uint16_info; - auto uint16_result = ShaderRecompiler::Recompile(shader, options); + auto uint16_result = RecompileForTest(shader, options); const auto uint16_source = DisassembleSpirvBinary(uint16_result.spirv); Check(Common::ContainsStr(uint16_source, "OpVariable %_ptr_Output_v4uint Output"), @@ -10022,7 +10072,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { ShaderPixelInputInfo unorm16_info; unorm16_info.target_output_mode[0] = 5; options.input_info.pixel = &unorm16_info; - auto unorm16_result = ShaderRecompiler::Recompile(shader, options); + auto unorm16_result = RecompileForTest(shader, options); const auto unorm16_source = DisassembleSpirvBinary(unorm16_result.spirv); Check(CountSourceOccurrences(unorm16_source, "UnpackUnorm2x16") == 2u, "compressed UNORM16 MRT export did not unpack two normalized pairs"); @@ -10037,7 +10087,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { }; ShaderPixelInputInfo default_info; options.input_info.pixel = &default_info; - auto partial_result = ShaderRecompiler::Recompile(partial_shader, options); + auto partial_result = RecompileForTest(partial_shader, options); const auto partial_source = DisassembleSpirvBinary(partial_result.spirv); Check(SpirvSourceHasInstructionOperand(partial_source, "OpBitcast", "%uint_1065353216"), @@ -10048,7 +10098,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { uint_info.target_output_mode[0] = 7; options.input_info.pixel = &uint_info; auto partial_uint_result = - ShaderRecompiler::Recompile(partial_shader, options); + RecompileForTest(partial_shader, options); const auto partial_uint_source = DisassembleSpirvBinary(partial_uint_result.spirv); Check(SpirvSourceHasInstructionOperand(partial_uint_source, @@ -10062,7 +10112,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { 0xbf810000u, }; auto compressed_ba_result = - ShaderRecompiler::Recompile(compressed_ba_shader, options); + RecompileForTest(compressed_ba_shader, options); const auto compressed_ba_source = DisassembleSpirvBinary(compressed_ba_result.spirv); Check(CountSourceOccurrences(compressed_ba_source, "OpCompositeExtract") == @@ -10075,7 +10125,7 @@ void TestNewShaderRecompilerExpPixelOutputs() { unorm16_info.target_output_mode[0] = 5; options.input_info.pixel = &unorm16_info; auto unorm16_ba_result = - ShaderRecompiler::Recompile(compressed_ba_shader, options); + RecompileForTest(compressed_ba_shader, options); const auto unorm16_ba_source = DisassembleSpirvBinary(unorm16_ba_result.spirv); Check(CountSourceOccurrences(unorm16_ba_source, "UnpackUnorm2x16") == 1u, @@ -10221,24 +10271,24 @@ void TestRenderTargetReverseExportMapping() { auto options = MakeCompileOptions(ShaderType::Pixel); options.input_info.pixel = &identity_info; - auto identity_result = ShaderRecompiler::Recompile(shader, options); + auto identity_result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(identity_result.spirv, 79u) == 0u, "identity MRT export unexpectedly added a component shuffle"); options.input_info.pixel = &reversed_info; - auto reversed_result = ShaderRecompiler::Recompile(shader, options); + auto reversed_result = RecompileForTest(shader, options); Check(SpirvContainsVectorShuffle(reversed_result.spirv, {3u, 2u, 1u, 0u}), "reverse MRT export did not emit a WZYX component shuffle"); CheckSpirvBinaryValidates(reversed_result.spirv); reversed_info.target_export_mapping[0] = gr32.export_mapping; - auto gr32_result = ShaderRecompiler::Recompile(shader, options); + auto gr32_result = RecompileForTest(shader, options); Check(SpirvContainsVectorShuffle(gr32_result.spirv, {1u, 0u, 2u, 3u}), "reverse GR32F MRT export did not emit a YXZW component shuffle"); CheckSpirvBinaryValidates(gr32_result.spirv); reversed_info.target_export_mapping[0] = argb.export_mapping; - auto argb_result = ShaderRecompiler::Recompile(shader, options); + auto argb_result = RecompileForTest(shader, options); Check(SpirvContainsVectorShuffle(argb_result.spirv, {3u, 0u, 1u, 2u}), "alternate-reversed MRT export did not emit an ARGB component shuffle"); CheckSpirvBinaryValidates(argb_result.spirv); @@ -10256,13 +10306,12 @@ void TestRenderTargetReverseExportMapping() { std::array mappings{}; mappings[0] = gr32.export_mapping; ShaderPixelInputInfo compiled_info{}; - ShaderVertexInputInfo vertex_info{}; - PrepareProgram(regs, sh, vertex_info, mappings, compiled_info); + PrepareProgram(regs, sh, mappings, compiled_info); Check(compiled_info.target_export_mapping[0].IsIdentity(), "inactive reverse MRT mapping was not normalized out of the shader " "cache key"); sh.target_output_mode[0] = 4; - PrepareProgram(regs, sh, vertex_info, mappings, compiled_info); + PrepareProgram(regs, sh, mappings, compiled_info); Check(compiled_info.target_export_mapping[0] == gr32.export_mapping, "active reverse MRT mapping was lost before shader specialization"); } @@ -10283,7 +10332,7 @@ void TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled() { auto options = MakeCompileOptions(ShaderType::Pixel); options.input_info.pixel = &ps_info; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(SpirvInstructionOpcodeCount(result.spirv, 252) != 0, "pixel valid-mask export should still lower to OpKill"); Check(!SpirvContainsExecutionMode(result.spirv, @@ -10298,7 +10347,7 @@ void TestNewShaderRecompilerEarlyZDisabledWhenPixelKillEnabled() { 0xbf810000u, }; ps_info.ps_pixel_kill_enable = false; - auto ordinary_result = ShaderRecompiler::Recompile(ordinary_shader, options); + auto ordinary_result = RecompileForTest(ordinary_shader, options); const auto ordinary_source = DisassembleSpirvBinary(ordinary_result.spirv); Check(SpirvInstructionOpcodeCount(ordinary_result.spirv, 252) == 0, "ordinary pixel shader unexpectedly contains OpKill"); @@ -10336,7 +10385,7 @@ void TestNewShaderRecompilerNativeBindingPlan() { options.dump_ir = true; options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); const auto *buffers = ShaderRecompiler::IR::FindBinding( result.program.bindings, BindingKind::Buffers); const ShaderRecompiler::IR::DescriptorBinding *sampled = nullptr; @@ -10392,46 +10441,17 @@ void TestNewShaderRecompilerNativeBindingPlan() { #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS ExpectFatal( [&] { - auto malformed = ShaderRecompiler::Recompile(shader, options); + auto malformed = RecompileForTest(shader, options); auto malformed_buffers = std::ranges::find_if( malformed.program.bindings.descriptors, [](const auto &binding) { return binding.kind == BindingKind::Buffers; - }); + }); malformed_buffers->resources.clear(); - (void)ShaderRecompiler::Spirv::EmitProgram( - malformed.program, malformed.resources, options.input_info); + (void)ShaderRecompiler::Spirv::EmitProgram(malformed.program, + options.input_info); }, "malformed native binding topology did not terminate SPIR-V emission"); - uint32_t float_storage = UINT32_MAX; - for (uint32_t i = 0; i < result.program.info.images.size(); i++) { - const auto &image = result.program.info.images[i]; - if (image.resource_class == - ShaderRecompiler::IR::ImageResourceClass::Storage && - image.numeric_class == Prospero::TextureNumericClass::Float) { - float_storage = i; - break; - } - } - Check(float_storage != UINT32_MAX, - "native validation fixture lacks float storage image"); - ExpectFatal( - [&] { - auto stale_resources = result.resources; - stale_resources.images[float_storage].dwords[1] = 20u << 20u; - (void)ShaderRecompiler::Spirv::EmitProgram( - result.program, stale_resources, options.input_info); - }, - "stale specialized image format did not terminate SPIR-V emission"); - ExpectFatal( - [&] { - auto stale_dimension = result.resources; - stale_dimension.images[float_storage].dwords[3] &= 0x0fffffffu; - (void)ShaderRecompiler::Spirv::EmitProgram( - result.program, stale_dimension, options.input_info); - }, - "stale specialized image dimension did not terminate SPIR-V emission"); - ShaderRecompiler::IR::Inst *buffer_handle = nullptr; for (auto *block : result.program.blocks) { const auto found = std::ranges::find_if(*block, [](const auto &inst) { @@ -10448,8 +10468,8 @@ void TestNewShaderRecompilerNativeBindingPlan() { ExpectFatal( [&] { buffer_handle->SetFlags(UINT32_MAX); - (void)ShaderRecompiler::Spirv::EmitProgram( - result.program, result.resources, options.input_info); + (void)ShaderRecompiler::Spirv::EmitProgram(result.program, + options.input_info); }, "invalid dense buffer resource did not terminate SPIR-V emission"); #endif @@ -10733,7 +10753,7 @@ void TestTypedDescriptorRealCarryAndScalarLoads() { auto user_data = ImageTestUserData(); auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(inline_sampler_shader, options); + auto result = RecompileForTest(inline_sampler_shader, options); } void TestSrtWalkerRealSmemTranslation() { @@ -11073,7 +11093,7 @@ void TestNewShaderRecompilerStageInputInfo() { auto cs_options = MakeCompileOptions(ShaderType::Compute); cs_options.input_info.compute = &cs_info; - auto cs_result = ShaderRecompiler::Recompile(shader, cs_options); + auto cs_result = RecompileForTest(shader, cs_options); Check(ProgramHasInput(cs_result.program, StageInputKind::WorkgroupId), "compute WorkgroupId input missing from reflection"); Check(ProgramHasInput(cs_result.program, StageInputKind::LocalInvocationId), @@ -11095,7 +11115,7 @@ void TestNewShaderRecompilerStageInputInfo() { auto vs_options = MakeCompileOptions(ShaderType::Vertex); - auto vs_result = ShaderRecompiler::Recompile(shader, vs_options); + auto vs_result = RecompileForTest(shader, vs_options); Check(ProgramHasInput(vs_result.program, StageInputKind::VertexIndex), "vertex VertexIndex input missing from reflection"); Check(ProgramHasInput(vs_result.program, StageInputKind::InstanceIndex), @@ -11116,7 +11136,7 @@ void TestNewShaderRecompilerStageInputInfo() { auto ps_options = MakeCompileOptions(ShaderType::Pixel); ps_options.input_info.pixel = &ps_info; - auto ps_result = ShaderRecompiler::Recompile(shader, ps_options); + auto ps_result = RecompileForTest(shader, ps_options); Check(ProgramHasInput(ps_result.program, StageInputKind::FragCoord), "pixel FragCoord input missing from reflection"); Check(ProgramInputCount(ps_result.program, StageInputKind::Parameter) == 2, @@ -11136,7 +11156,7 @@ void TestNewShaderRecompilerStageInputInfo() { SetIdentityInterpolatorSettings(&ps_pos_y_info); ps_options.input_info.pixel = &ps_pos_y_info; - auto ps_pos_y_result = ShaderRecompiler::Recompile(shader, ps_options); + auto ps_pos_y_result = RecompileForTest(shader, ps_options); Check(ProgramHasInput(ps_pos_y_result.program, StageInputKind::FragCoord), "pixel POS_Y-only FragCoord input missing from reflection"); Check(SpirvHasDecorationValue(ps_pos_y_result.spirv, 11u, 15u), @@ -11157,9 +11177,8 @@ void TestNewShaderRecompilerPixelPipelineEntry() { HW::ShaderRegisters sh{}; const std::array mappings{}; - ShaderVertexInputInfo vertex_info{}; ShaderPixelInputInfo input_info{}; - const auto params = PrepareProgram(regs, sh, vertex_info, mappings, input_info); + const auto params = PrepareProgram(regs, sh, mappings, input_info); Check(params.hash == XXH3_64bits(params.code.data(), params.code.size_bytes()) && params.code.data() == shader, "pixel shader program parameters lost source identity"); @@ -11187,8 +11206,7 @@ void TestNewShaderRecompilerPixelPipelineEntry() { ShaderMapUserData(vcc_regs.ps_regs.data_addr, vcc_mapped); ShaderPixelInputInfo vcc_input{}; - const auto vcc_params = - PrepareProgram(vcc_regs, sh, vertex_info, mappings, vcc_input); + const auto vcc_params = PrepareProgram(vcc_regs, sh, mappings, vcc_input); CompilePixelRuntime(vcc_params, vcc_input); } @@ -11221,7 +11239,7 @@ void TestComputeLdsAllocationIdentity() { options.input_info.compute = &input_info; options.wave_size = input_info.wave_size; options.scratch_dwords = input_info.scratch_size_dwords; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(MeasureSpirv(result.spirv).workgroup_variables == 1u, "LDS shader did not declare exactly one Workgroup variable"); Check(SpirvArrayLengthCount(result.spirv, expected_dwords) == 1u, @@ -11288,7 +11306,7 @@ void TestComputeLdsAllocationIdentity() { scratch_options.wave_size = scratch_info.wave_size; scratch_options.scratch_dwords = scratch_info.scratch_size_dwords; auto scratch_result = - ShaderRecompiler::Recompile(scratch_params.code, scratch_options); + RecompileForTest(scratch_params.code, scratch_options); Check(scratch_result.program.scratch_dwords == 7, "AGC per-thread scratch size did not reach the compiler program"); Check(MakeStageStaticKey(scratch_info) != MakeStageStaticKey(lds_896), @@ -11305,7 +11323,7 @@ void TestComputeLdsAllocationIdentity() { auto append_options = MakeCompileOptions(ShaderType::Compute); append_options.input_info.compute = &append_info; auto append_result = - ShaderRecompiler::Recompile(append_shader, append_options); + RecompileForTest(append_shader, append_options); Check(SpirvUnsignedLessThanBoundCount(append_result.spirv, 1152u) == 1u, "typed LDS append omitted the declared allocation bound"); CheckSpirvBinaryValidates(append_result.spirv); @@ -11330,7 +11348,7 @@ void TestWave64LdsSynchronization() { auto options = MakeCompileOptions(ShaderType::Compute); options.wave_size = wave_size; options.input_info.compute = &input_info; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); return result; }; @@ -11450,10 +11468,6 @@ void TestPixelProgramCacheBindingIdentity() { depth_export.ps_depth_export_enable = true; Check(MakeStageStaticKey(no_depth_export) != MakeStageStaticKey(depth_export), "pixel shader identity omitted depth-export semantics"); - auto shifted_push = no_depth_export; - shifted_push.push_constant_offset = 36; - Check(MakeStageStaticKey(no_depth_export) != MakeStageStaticKey(shifted_push), - "pixel shader identity omitted graphics push-bank placement"); auto check_program_identity = [&](const uint32_t *shader) { HW::PixelShaderInfo regs{}; @@ -11463,23 +11477,22 @@ void TestPixelProgramCacheBindingIdentity() { ShaderMapUserData(regs.ps_regs.data_addr, mapped); const std::array identity_mappings{}; - ShaderVertexInputInfo vertex_info{}; ShaderPixelInputInfo first_info{}; - const auto first_params = - PrepareProgram(regs, sh, vertex_info, identity_mappings, first_info); + const auto first_params = PrepareProgram(regs, sh, identity_mappings, + first_info); const auto first_key = MakeStageStaticKey(first_info); CompilePixelRuntime(first_params, first_info); ShaderPixelInputInfo second_info{}; - const auto second_params = - PrepareProgram(regs, sh, vertex_info, identity_mappings, second_info); + const auto second_params = PrepareProgram(regs, sh, identity_mappings, + second_info); + CompilePixelRuntime(second_params, second_info); Check( first_params.hash == second_params.hash && first_key == MakeStageStaticKey(second_info) && first_info.stage.program != nullptr && - MaterializeProgram(first_info.stage.program, second_params, second_info) && - second_info.stage.program == first_info.stage.program, - "pixel program matching did not reuse an identical graphics placement"); + second_info.stage.program != nullptr, + "pixel program matching did not preserve an identical source/static identity"); return std::pair {first_params.hash, first_key}; }; @@ -11488,59 +11501,6 @@ void TestPixelProgramCacheBindingIdentity() { Check(request_01 == request_10, "relocated identical pixel programs did not share their source identity"); - const uint32_t push_shader[] = { - EncodeVop1(0x01, 0, 0), EncodeVop1(0x01, 1, 1), - EncodeVop1(0x01, 2, 2), EncodeVop1(0x01, 3, 3), - EncodeExp0(0x00, 0xf), EncodeExp1(0, 1, 2, 3), 0xbf810000u, - }; - HW::PixelShaderInfo push_regs{}; - push_regs.ps_regs.data_addr = reinterpret_cast(push_shader); - push_regs.ps_regs.rsrc2.user_sgpr = 4; - ShaderMappedData push_mapped{}; - push_mapped.code_size_bytes = sizeof(push_shader); - ShaderMapUserData(push_regs.ps_regs.data_addr, push_mapped); - HW::ShaderRegisters push_sh{}; - push_sh.target_output_mode[0] = 4; - const std::array mappings{}; - const auto MakeVertex = [](uint32_t push_size) { - ShaderVertexInputInfo info{}; - auto program = std::make_shared(); - program->stage = ShaderType::Vertex; - program->bindings.push_constant_size = push_size; - info.stage.program = std::move(program); - return info; - }; - const auto vertex_at_zero = MakeVertex(0); - const auto vertex_at_36 = MakeVertex(36); - ShaderPixelInputInfo at_zero{}; - ShaderPixelInputInfo at_36{}; - ShaderPixelInputInfo at_zero_again{}; - const auto params_at_zero = - PrepareProgram(push_regs, push_sh, vertex_at_zero, mappings, at_zero); - const auto key_at_zero = MakeStageStaticKey(at_zero); - CompilePixelRuntime(params_at_zero, at_zero); - const auto params_at_36 = - PrepareProgram(push_regs, push_sh, vertex_at_36, mappings, at_36); - const auto key_at_36 = MakeStageStaticKey(at_36); - CompilePixelRuntime(params_at_36, at_36); - const auto params_at_zero_again = - PrepareProgram(push_regs, push_sh, vertex_at_zero, mappings, - at_zero_again); - auto shifted_options = MakeCompileOptions(ShaderType::Pixel); - shifted_options.input_info.pixel = &at_36; - shifted_options.push_constant_offset = at_36.push_constant_offset; - auto shifted_result = - ShaderRecompiler::Recompile(push_shader, shifted_options); - Check(at_zero.push_constant_offset == 0 && - at_36.push_constant_offset == 36 && - key_at_zero != key_at_36 && - MakeStageStaticKey(at_zero_again) == key_at_zero && - at_zero.stage.program != at_36.stage.program && - MaterializeProgram(at_zero.stage.program, params_at_zero_again, - at_zero_again) && - at_zero_again.stage.program == at_zero.stage.program && - SpirvHasMemberDecorationValue(shifted_result.spirv, 35u, 36u), - "pixel program matching did not partition and reuse graphics push placement"); } void TestGraphicsPushConstantPlacement() { @@ -11558,26 +11518,18 @@ void TestGraphicsPushConstantPlacement() { auto options = MakeCompileOptions(ShaderType::Pixel); options.input_info.pixel = &pixel_info; options.user_data = user_data; - options.push_constant_offset = 36; - auto placed = ShaderRecompiler::Recompile(shader, options); - Check(placed.program.bindings.push_constant_offset == 36 && - placed.program.bindings.push_constant_size == sizeof(user_data) && + constexpr uint32_t PushDataStart = 7; + auto placed = + RecompileForTest(shader, options, nullptr, nullptr, PushDataStart); + Check(placed.program.bindings.UsesPushData() && + placed.program.bindings.push_data_start_dword == PushDataStart && ShaderRecompiler::IR::FindBinding(placed.program.bindings, - BindingKind::UserData) == nullptr, - "pixel push constants did not follow the vertex payload"); - Check(SpirvHasMemberDecorationValue(placed.spirv, OffsetDecoration, 36), - "SPIR-V push block omitted the graphics-bank member offset"); + BindingKind::ShaderData) == nullptr, + "pixel shader did not use its assigned shared push-data position"); + Check(SpirvHasMemberDecorationValue(placed.spirv, OffsetDecoration, 0), + "SPIR-V push-data block did not use the canonical zero base"); CheckSpirvBinaryValidates(placed.spirv); - - options.push_constant_offset = 124; - auto spilled = ShaderRecompiler::Recompile(shader, options); - Check(spilled.program.bindings.push_constant_offset == 124 && - spilled.program.bindings.push_constant_size == 0 && - ShaderRecompiler::IR::FindBinding(spilled.program.bindings, - BindingKind::UserData) != nullptr, - "pixel data that crossed the graphics push bank did not spill"); - CheckSpirvBinaryValidates(spilled.spirv); } void TestNewShaderRecompilerUnsupportedMemoryDecode() { @@ -11621,7 +11573,7 @@ void TestNewShaderRecompilerFlatUserPointerUsesDma() { auto options = MakeCompileOptions(ShaderType::Pixel); options.user_data = user_data; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); Check(result.program.info.uses_dma, "FLAT user pointer did not enable DMA"); CheckSpirvBinaryValidates(result.spirv); @@ -11642,7 +11594,7 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() { auto options = MakeCompileOptions(ShaderType::Compute); options.user_data = user_data; options.scratch_dwords = 1; - auto result = ShaderRecompiler::Recompile(segmented_shader, options); + auto result = RecompileForTest(segmented_shader, options); Check(result.program.info.uses_dma, "GLOBAL null-SADDR did not enable DMA"); Check(Common::ContainsStr(result.ir_dump, "GetScratchResource") && @@ -11657,7 +11609,7 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { auto options = MakeCompileOptions(stage); options.dump_ir = true; - auto result = ShaderRecompiler::Recompile(shader, options); + auto result = RecompileForTest(shader, options); CheckSpirvBinaryValidates(result.spirv); CheckSpirvBudget(name, result.spirv, budget); return result; @@ -11828,7 +11780,7 @@ void TestNewShaderRecompilerSpirvSizeBaselines() { wide_gds_result.program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr, "live native wide GDS access did not allocate its descriptor binding"); - const auto count_shared = [](const ShaderRecompiler::CompileResult &result, + const auto count_shared = [](const auto &result, ShaderRecompiler::IR::ValueOpcode opcode) { uint32_t count = 0; for (const auto *block : result.program.blocks) {