diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp index 6f246a66e..6cc4ef92b 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp @@ -61,10 +61,10 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR:: : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value); } -ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) { +ImageSampleLayout Layout(const IR::MemoryInfo& mem) { ImageSampleLayout layout; uint32_t cursor = 0; - const auto& info = ImageDimensionInfoFor(dimension); + const auto& info = ImageDimensionInfoFor(mem.image_dimension); if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++; if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++; if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++; @@ -80,48 +80,6 @@ ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) { return layout; } -void ValidateIndirectImageOperands(ValueEmitContext& ctx, const IR::Inst& inst) { - const auto op = inst.GetOpcode(); - const auto& memory = ctx.Memory(inst); - const auto& images = ctx.state.program.info.images; - const auto& root = images.at(memory.resource); - if (root.indirect_root != memory.resource) return; - if (op != IR::ValueOpcode::ImageSampleRaw && - op != IR::ValueOpcode::ImageGatherRaw && op != IR::ValueOpcode::ImageRead) { - ctx.Fail(inst, "requires an unsupported indirect image operation"); - } - const auto& native = ImageDimensionInfoFor(memory.image_dimension); - const auto native_layout = Layout(memory, memory.image_dimension); - const bool sampling = op == IR::ValueOpcode::ImageSampleRaw || - op == IR::ValueOpcode::ImageGatherRaw; - const auto validate = [&](const IR::ImageResource& image) { - const auto& candidate = ImageDimensionInfoFor(image.dimension); - // Sample/fetch ignore the native array slice for a non-array T# while - // keeping the original positions of later operands. - const bool projected_2d = memory.image_dimension == ImageDimension::Dim2DArray && - image.dimension == ImageDimension::Dim2D; - bool compatible = projected_2d || - (candidate.coordinate_components == native.coordinate_components && - candidate.multisampled == native.multisampled); - if (sampling) { - compatible &= candidate.multisampled == 0u && native.multisampled == 0u; - compatible &= projected_2d || Layout(memory, image.dimension) == native_layout; - if (HasFlag(memory, Decoder::ImageSampleFlagDerivative)) { - compatible &= candidate.spatial_components == native.spatial_components; - } - if (op == IR::ValueOpcode::ImageGatherRaw || - HasFlag(memory, Decoder::ImageSampleFlagCompare)) { - compatible &= image.dimension != ImageDimension::Dim3D; - } - } - if (!compatible) ctx.Fail(inst, "has incompatible image operand layout"); - }; - validate(root); - for (const auto resource: root.indirect_resources) { - if (resource != memory.resource) validate(images.at(resource)); - } -} - uint32_t ZeroF32(EmitterState& state) { return ConstantF32(state, 0); } @@ -646,45 +604,6 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d return PackImageTexel(ctx, mem, texel); } -struct ImageSampler { - uint32_t id = 0; - bool dynamic = false; -}; - -auto EmitImageSamplers(ValueEmitContext& ctx, const IR::Inst& inst) { - auto& state = ctx.state; - const auto& mem = ctx.Memory(inst); - const auto& images = state.program.info.images; - const auto& samplers = state.program.info.samplers; - std::array(Prospero::TextureNumericClass::Sint) + 1u> result {}; - const auto add = [&](uint32_t image, uint32_t sampler) { - auto& entry = result[static_cast(images[image].numeric_class)]; - if (entry.id != 0u) return; - const auto& choices = samplers.at(sampler).indirect_resources; - uint32_t index = 0; - if (!choices.empty()) { - const auto* handle = inst.Arg(1).ResolveInstruction(); - if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetSamplerResource) - ctx.Fail(inst, "has invalid finite sampler key provenance"); - index = EmitIndexSwitch( - state, ctx.Def(handle->Arg(0)), static_cast(choices.size()), - TypeU32(state), [&](uint32_t ordinal) { return ConstantU32(state, choices[ordinal]); }); - } - entry = {LoadSamplerDescriptor(state, sampler, index), index != 0u}; - }; - add(mem.resource, mem.sampler); - if (images[mem.resource].indirect_root == mem.resource) { - for (const auto& pair: state.program.info.sampled_pairs) - if (images[pair.image].indirect_root == mem.resource && - samplers[pair.sampler].snapshot_index == samplers[mem.sampler].snapshot_index) - add(pair.image, pair.sampler); - for (const auto candidate: images[mem.resource].indirect_resources) - if (result[static_cast(images[candidate].numeric_class)].id == 0u) - ctx.Fail(inst, "has no sampler variant for an indirect image candidate"); - } - return result; -} - template uint32_t EmitImageAccess(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t result_type, const Emit& emit) { @@ -778,7 +697,6 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { const auto& mem = ctx.Memory(inst); const auto image_arg = inst.Arg(0); ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource); - ValidateIndirectImageOperands(ctx, inst); const auto& image = state.program.info.images.at(mem.resource); const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1)); if (op == IR::ValueOpcode::ImageQueryDimensions) { @@ -811,14 +729,15 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { return; } if (op == IR::ValueOpcode::ImageRead) { + const auto numeric_class = image.numeric_class; const auto condition = ctx.Arg(inst, 2); ctx.Define( inst, EmitValueOrDefaultIfCondition( state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4), [&]() { - return EmitImageAccess( - ctx, inst, TypeU32Vector(state, 4), + const auto color = EmitImageAccess( + ctx, inst, ImageVectorType(state, numeric_class, 4), [&](uint32_t resource, uint32_t array_index) { const auto& candidate = state.program.info.images[resource]; const auto dimension = candidate.dimension; @@ -837,9 +756,10 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { descriptor, coord, spv::ImageOperandsLodMask, LodU32(ctx, mem, *address, mem.image_dimension)); } - return ResultVector(ctx, UnpackImageTexel(ctx, mem, fetched), - candidate.numeric_class, false, mem); + return fetched; }); + return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class, + false, mem); })); return; } @@ -860,7 +780,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) { const auto dimension = image.dimension; const auto& dimension_info = ImageDimensionInfoFor(dimension); - const auto layout = Layout(mem, mem.image_dimension); + const auto layout = Layout(mem); const auto numeric_class = image.numeric_class; const bool dref = HasFlag(mem, Decoder::ImageSampleFlagCompare); IR::Value offset_value; @@ -894,6 +814,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { ctx.Fail(inst, "has an unsupported 1D-array gather"); return; } + const auto result_numeric_class = dref ? Prospero::TextureNumericClass::Float + : numeric_class; + const auto result_type = ImageVectorType(state, result_numeric_class, 4); uint32_t component_or_dref; if (dref) { component_or_dref = layout.dref != NoImageComponent @@ -902,13 +825,9 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { } else { component_or_dref = ConstantU32(state, ImageGatherSource(state, mem)); } - const auto samplers = EmitImageSamplers(ctx, inst); + const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler); const auto EmitGather = [&](uint32_t resource, uint32_t array_index) { const auto& candidate = state.program.info.images[resource]; - const auto numeric_class = dref ? Prospero::TextureNumericClass::Float - : candidate.numeric_class; - const auto result_type = ImageVectorType(state, numeric_class, 4); - const auto sampler = samplers[static_cast(candidate.numeric_class)]; const auto dimension = candidate.dimension; const auto coord = CoordF32( ctx, mem, *address, layout.coord, @@ -922,8 +841,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { offset = PackedOffset(ctx, offset_value, dimension, operand_mask); } const auto EmitMip = [&](uint32_t mip) { - const auto sampled = MakeSampledImage(state, resource, sampler.id, mip, array_index, - sampler.dynamic); + const auto sampled = MakeSampledImage(state, resource, sampler_id, mip, array_index); const auto sample = state.builder.AllocateId(); const std::array operands {operand_mask, offset}; state.builder.AddFunction( @@ -932,15 +850,15 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { std::span(operands).first(operand_mask != 0u ? 2u : 0u)); return sample; }; - const auto gathered = candidate.mip_mode == IR::ImageMipMode::Dynamic && candidate.mip_count > 1u + return candidate.mip_mode == IR::ImageMipMode::Dynamic && candidate.mip_count > 1u ? EmitIndexSwitch( state, GatherMip(ctx, inst, mem, *address, layout, candidate.mip_count), candidate.mip_count, result_type, EmitMip) : EmitMip(0u); - return ResultVector(ctx, UnpackImageGather(ctx, mem, gathered), numeric_class, - false, mem, true); }; - ctx.Define(inst, EmitImageAccess(ctx, inst, TypeU32Vector(state, 4), EmitGather)); + const auto sample = EmitImageAccess(ctx, inst, result_type, EmitGather); + ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample), + result_numeric_class, false, mem, true)); return; } const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) || @@ -953,8 +871,10 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { } else if (dref) { opcode = spv::OpImageSampleDrefImplicitLod; } + uint32_t result_type = ImageVectorType(state, numeric_class, 4); uint32_t dref_value = 0; if (dref) { + result_type = TypeF32(state); dref_value = ZeroF32(state); if (layout.dref != NoImageComponent) { dref_value = AddressF32(ctx, mem, *address, layout.dref); @@ -980,16 +900,23 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { operands[1] = AddressF32(ctx, mem, *address, layout.bias); operand_count = 2; } + const auto& samplers = state.program.info.samplers.at(mem.sampler).indirect_resources; + uint32_t sampler_index = 0; + if (!samplers.empty()) { + const auto* handle = inst.Arg(1).ResolveInstruction(); + if (handle == nullptr || handle->GetOpcode() != IR::ValueOpcode::GetSamplerResource) + ctx.Fail(inst, "has invalid finite sampler key provenance"); + sampler_index = EmitIndexSwitch( + state, ctx.Def(handle->Arg(0)), static_cast(samplers.size()), + TypeU32(state), [&](uint32_t ordinal) { return ConstantU32(state, samplers[ordinal]); }); + } if (layout.offset != NoImageComponent && !offset_value.IsImmediate() && !state.maintenance8_enabled) { ctx.Fail(inst, "requires maintenance8 for a dynamic sample texel offset"); } - const auto samplers = EmitImageSamplers(ctx, inst); + const auto sampler_id = LoadSamplerDescriptor(state, mem.sampler, sampler_index); const auto EmitSample = [&](uint32_t resource, uint32_t array_index) { const auto& candidate = state.program.info.images[resource]; - const auto result_type = dref ? TypeF32(state) - : ImageVectorType(state, candidate.numeric_class, 4); - const auto sampler = samplers[static_cast(candidate.numeric_class)]; auto sample_operands = operands; auto sample_operand_count = operand_count; if (layout.offset != NoImageComponent) { @@ -1001,16 +928,19 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) { CoordF32(ctx, mem, *address, layout.coord, ImageDimensionInfoFor(candidate.dimension).coordinate_components, candidate.cube); - const auto sampled = MakeSampledImage(state, resource, sampler.id, 0u, array_index, - sampler.dynamic); + const auto sampled = MakeSampledImage(state, resource, sampler_id, 0u, array_index, + sampler_index != 0u); const auto sample = state.builder.AllocateId(); state.builder.AddFunction(opcode, result_type, sample, sampled, coord, std::span(&dref_value, dref ? 1u : 0u), std::span(sample_operands).first(sample_operand_count)); - return ResultVector(ctx, dref ? sample : UnpackImageTexel(ctx, mem, sample), - candidate.numeric_class, dref, mem); + return sample; }; - ctx.Define(inst, EmitImageAccess(ctx, inst, TypeU32Vector(state, 4), EmitSample)); + auto result = EmitImageAccess(ctx, inst, result_type, EmitSample); + if (!dref) { + result = UnpackImageTexel(ctx, mem, result); + } + ctx.Define(inst, ResultVector(ctx, result, numeric_class, dref, mem)); return; } if (image_info.access == IR::ImageAccess::Atomic) { diff --git a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h index 9e8f8b74d..e3611f0d8 100644 --- a/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h +++ b/src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h @@ -272,8 +272,6 @@ struct ImageSampleLayout { uint32_t lod = NoImageComponent; uint32_t grad_x = NoImageComponent; uint32_t grad_y = NoImageComponent; - - bool operator==(const ImageSampleLayout&) const = default; }; struct F32Class { diff --git a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp index 4fd5baea7..dc530350f 100644 --- a/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp +++ b/src/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp @@ -648,6 +648,10 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna return SpecializationFail("indirect image specialization has no typed candidate"); } const auto& image_class = specialization.images[exemplar]; + const auto is_2d = [](Decoder::ImageDimension dimension) { + return dimension == Decoder::ImageDimension::Dim2D || + dimension == Decoder::ImageDimension::Dim2DArray; + }; for (uint32_t candidate = 0; candidate < specialization.images.size(); candidate++) { auto& image = specialization.images[candidate]; if (image.indirect_root != root_index) { @@ -661,7 +665,11 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna image.shader_swizzle = image_class.shader_swizzle; image.cube = image_class.cube; } - if (image.mip_count != image_class.mip_count || + const bool same_coordinates = image.dimension == image_class.dimension && + image.cube == image_class.cube; + if (image.numeric_class != image_class.numeric_class || + (!same_coordinates && !(is_2d(image.dimension) && is_2d(image_class.dimension))) || + image.mip_count != image_class.mip_count || image.conversion_format != image_class.conversion_format || image.shader_swizzle != image_class.shader_swizzle) { return SpecializationFail( @@ -1309,20 +1317,9 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& } SamplerPlan sampler_plan; + EXIT_IF(!BuildSamplerPlan(program.info, sampler_plan)); auto& samplers = program.info.samplers; auto& sampled_pairs = program.info.sampled_pairs; - const auto original_pair_count = sampled_pairs.size(); - for (size_t index = 0; index < original_pair_count; ++index) { - const auto pair = sampled_pairs[index]; - uint32_t classes = 1u << static_cast(images[pair.image].numeric_class); - for (const auto candidate: images[pair.image].indirect_resources) { - const auto bit = 1u << static_cast(images[candidate].numeric_class); - if ((classes & bit) != 0u) continue; - classes |= bit; - sampled_pairs.push_back({candidate, pair.sampler, pair.first_use_pc}); - } - } - EXIT_IF(!BuildSamplerPlan(program.info, sampler_plan)); const auto original_sampler_count = samplers.size(); samplers.reserve(sampler_plan.sampler_count); for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) { @@ -1434,6 +1431,10 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& memory.sampler = sampler_plan.mapping[memory.sampler][type]; EXIT_IF(memory.sampler == UINT32_MAX); } + EXIT_IF(image.indirect_root == memory.resource && + inst.GetOpcode() != ValueOpcode::ImageSampleRaw && + inst.GetOpcode() != ValueOpcode::ImageGatherRaw && + inst.GetOpcode() != ValueOpcode::ImageRead); } } for (auto& memory: memory_info) { diff --git a/tests/ResourceTrackingTests.cpp b/tests/ResourceTrackingTests.cpp index c9c4c7fc6..a5f7e592b 100644 --- a/tests/ResourceTrackingTests.cpp +++ b/tests/ResourceTrackingTests.cpp @@ -434,18 +434,8 @@ void TestBoundedImageViewEligibility() { plan.descriptor_sources.push_back(source); } plan.descriptor_sources[image_source].indirect_descriptor->sources = std::move(selected); - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.images.size() == 2u && - snapshot.images[0].dwords == descriptor(0x30u, Type::kColor2D) && - snapshot.images[1].dwords == descriptor(0x30u, Type::kColor3D) && - specialization.images[0].dimension == Decoder::ImageDimension::Dim2D && - specialization.images[1].dimension == Decoder::ImageDimension::Dim3D, - "explicitly selected image dimensions were discarded before operand validation"); - const auto converted = plan.descriptor_sources[image_source].indirect_descriptor->sources.back(); - plan.descriptor_sources[converted].dwords[1] = Value( - static_cast(Libs::Graphics::Prospero::BufferFormat::k11_11_10UInt) << 20u); Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "explicitly selected incompatible conversion was silently normalized to null"); + "explicitly selected incompatible descriptor was silently normalized to null"); } void TestWaterfallImageTable() { @@ -1405,52 +1395,6 @@ void TestDescendingImageRange() { check_bitmap({0u, 127u}, 0u); } -void TestMixedIndirectImageSamplers() { - auto fixture = MakeIndirectImageFixture(false, 32u, 0u, 0u); - fixture->PlanAndTrack(); - auto& samplers = fixture->program.info.samplers; - samplers.resize(3u, samplers[0]); - samplers[0].indirect_resources = {1u, 2u}; - const auto plan = ExtractResourcePlan(fixture->program); - std::array data {0x3000u, 16u << 16u, 1u, 0u, - 0x2000u, 32u << 16u, 3u, 0u}; - LinearTestMemory memory; - using F = Libs::Graphics::Prospero::BufferFormat; - for (uint32_t record = 0; record < 3u; ++record) { - const auto offset = (0x1000u + record * 32u) / 4u; - memory.words[offset] = 0x100u + record; - memory.words[offset + 1u] = static_cast( - record == 0u ? F::k32_32_32_32Float : F::k32_32_32_32UInt) << 20u; - memory.words[offset + 3u] = Libs::Graphics::DstSel(4, 5, 6, 7) | - (uint32_t(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u); - } - const SrtRuntime runtime {.user_data = data, .userdata = &memory, - .read_specialization_memory = ReadLinearTestMemory}; - ResourceSnapshot snapshot; - ResourceSpecialization specialization; - Check(MaterializeResources(plan, runtime, snapshot, specialization) && snapshot.images.size() == 4u, - "mixed float/uint image table failed materialization"); - ApplyResourceSpecialization(fixture->program, specialization); - const auto& info = fixture->program.info; - Check(info.sampled_pairs.size() == 2u && info.samplers.size() == 6u, - "indirect image candidates duplicated sampler pairs or lost finite class variants"); - for (const auto& pair : info.sampled_pairs) { - const auto integer = info.images[pair.image].numeric_class == - Libs::Graphics::Prospero::TextureNumericClass::Uint; - const auto& sampler = info.samplers[pair.sampler]; - Check(sampler.snapshot_index == 0u && sampler.integer_border == integer && - sampler.force_point_filtering == integer && sampler.indirect_resources.size() == 2u && - pair.first_use_pc == 0x10f0u, - "mixed image class changed native sampler identity or integer-border semantics"); - for (uint32_t ordinal = 0; ordinal < sampler.indirect_resources.size(); ++ordinal) { - const auto& candidate = info.samplers[sampler.indirect_resources[ordinal]]; - Check(candidate.snapshot_index == ordinal + 1u && - candidate.integer_border == integer && candidate.force_point_filtering == integer, - "mixed image class mapped a finite sampler choice to the wrong variant"); - } - } -} - void TestUniformizedMaterialImageKeys() { namespace CFG = Libs::Graphics::ShaderRecompiler::CFG; enum class Variant { @@ -4607,7 +4551,6 @@ int main() { Run("scalar buffer selector domain", TestScalarBufferSelectorDomain); Run("shared append selector bounds", TestSharedAppendSelectorBounds); Run("descending image range", TestDescendingImageRange); - Run("mixed indirect image samplers", TestMixedIndirectImageSamplers); Run("SRT runtime", TestSrtFlatteningAndRuntimeMemoization); Run("dynamic SRT", TestDynamicSrtReadRemainsExplicit); Run("writable descriptor phi", TestWritableDescriptorPhi); diff --git a/tests/ShaderRecompilerComputeTests.cpp b/tests/ShaderRecompilerComputeTests.cpp index 0e89201bb..eff660eb3 100644 --- a/tests/ShaderRecompilerComputeTests.cpp +++ b/tests/ShaderRecompilerComputeTests.cpp @@ -1987,7 +1987,6 @@ public: vk::ImageLayout layout = vk::ImageLayout::eUndefined; u32 width = 0; u32 height = 0; - u32 depth = 1; u32 layers = 1; u32 mip_levels = 1; u32 dwords_per_pixel = 0; @@ -15452,12 +15451,11 @@ public: u32 view_layers = 0, vk::SampleCountFlagBits samples = vk::SampleCountFlagBits::e1, vk::Format view_format = vk::Format::eUndefined, - u32 view_base_mip = 0, u32 depth = 1) { + u32 view_base_mip = 0) { Image ret; ret.format = format; ret.width = width; ret.height = height; - ret.depth = depth; ret.layers = layers; ret.mip_levels = std::max(static_cast(initial_mips.size()), 1u); ret.dwords_per_pixel = dwords_per_pixel; @@ -15468,7 +15466,7 @@ public: image_info.format = format; image_info.extent.width = width; image_info.extent.height = height; - image_info.extent.depth = depth; + image_info.extent.depth = 1; image_info.mipLevels = ret.mip_levels; image_info.arrayLayers = layers; image_info.samples = samples; @@ -15533,13 +15531,13 @@ public: size_t total_dwords = 0; for (u32 level = 0; level < ret.mip_levels; level++) { total_dwords += - ImageMipDwordCount(width, height, dwords_per_pixel, level, layers) * MipExtent(depth, level); + ImageMipDwordCount(width, height, dwords_per_pixel, level, layers); } std::vector contents(total_dwords, 0); size_t offset = 0; for (u32 level = 0; level < ret.mip_levels; level++) { const auto level_dwords = - ImageMipDwordCount(width, height, dwords_per_pixel, level, layers) * MipExtent(depth, level); + ImageMipDwordCount(width, height, dwords_per_pixel, level, layers); const auto &src = initial_mips[level]; for (size_t i = 0; i < src.size() && i < level_dwords; i++) { contents[offset + i] = src[i]; @@ -15683,8 +15681,7 @@ public: if (image.resource_class == ShaderRecompiler::IR::ImageResourceClass::Sampled) { supported = supported || image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D || - image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray || - image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D; + image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray; } Require(test.name, "dispatch", supported, "unsupported image dimension needs a matching Vulkan test view"); @@ -19205,11 +19202,11 @@ private: copy.imageSubresource.layerCount = image->layers; copy.imageExtent.width = MipExtent(image->width, level); copy.imageExtent.height = MipExtent(image->height, level); - copy.imageExtent.depth = MipExtent(image->depth, level); + copy.imageExtent.depth = 1; copies.push_back(copy); offset += static_cast( ImageMipDwordCount(image->width, image->height, - image->dwords_per_pixel, level, image->layers) * MipExtent(image->depth, level) * + image->dwords_per_pixel, level, image->layers) * sizeof(u32)); } cmd.copyBufferToImage(staging, image->image, @@ -36137,9 +36134,6 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { program.info.images[root.indirect_resources[103]].dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; program.info.images[root.indirect_resources[103]].cube = true; - program.memory_info[0].image_dimension = - ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; - program.memory_info[0].image_address_components = 4; program.shader_info_complete = false; CollectShaderInfo(program, {.compute = &compute}); spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); @@ -36148,6 +36142,9 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { CountText(text, "OpImageSampleExplicitLod") == 3, "one cube split the native arrays into per-image samples"); + program.memory_info[0].image_dimension = + ShaderRecompiler::Decoder::ImageDimension::Dim2DArray; + program.memory_info[0].image_address_components = 4; for (u32 component = 0; component < 4u; component++) { constexpr std::array values{1.375f, 1.625f, 10.0f, 2.0f}; address.SetArg(component, Value(std::bit_cast(values[component]))); @@ -36201,83 +36198,6 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { Require(name, "mixed sample count", samples == 3u, "mixed candidate switch did not retain every image"); } -#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS - enum class RejectedAccess { LodPosition, Derivatives, Gather, Depth, Multisampled, Query, Write, Atomic }; - for (const auto [rejected, label] : { - std::pair{RejectedAccess::LodPosition, "IndirectImageLodPositionRejected"}, - std::pair{RejectedAccess::Derivatives, "IndirectImageDerivativeLayoutRejected"}, - std::pair{RejectedAccess::Gather, "IndirectImageVolumeGatherRejected"}, - std::pair{RejectedAccess::Depth, "IndirectImageVolumeDrefRejected"}, - std::pair{RejectedAccess::Multisampled, "IndirectImageMsaaSampleRejected"}, - std::pair{RejectedAccess::Query, "IndirectImageQueryRejected"}, - std::pair{RejectedAccess::Write, "IndirectImageWriteRejected"}, - std::pair{RejectedAccess::Atomic, "IndirectImageAtomicRejected"}}) { - ExpectFatal(label, [&] { - using Dimension = ShaderRecompiler::Decoder::ImageDimension; - for (auto &resource : program.info.images) { - resource.dimension = Dimension::Dim2DArray; - resource.cube = false; - } - program.info.images[2].dimension = Dimension::Dim3D; - auto &mem = program.memory_info[0]; - const auto position = std::ranges::find_if(*block, [&](const Inst &inst) { return &inst == &sample; }); - if (rejected == RejectedAccess::LodPosition) { - // Four available operands do not make native x,y,LOD compatible with x,y,z,LOD. - mem.image_dimension = Dimension::Dim2D; - program.info.images[0].dimension = program.info.images[1].dimension = Dimension::Dim2D; - } else if (rejected == RejectedAccess::Derivatives) { - mem.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagDerivative; - mem.image_address_components = 9u; // Capacity alone must not change the native gradient layout. - } else if (rejected == RejectedAccess::Gather) { - mem.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLevelZero; - mem.image_address_components = 3u; - const auto gather = block->PrependNewInst(position, ValueOpcode::ImageGatherRaw, - {Value(&image), Value(&sampler), Value(&address), Value(0u), Value(0u), - Value(0u), Value(0u), Value(true)}, memory_flag_bits); - sample.ReplaceUsesWith(Value(&*gather), false); - } else if (rejected == RejectedAccess::Depth) { - mem.image_sample_flags |= ShaderRecompiler::Decoder::ImageSampleFlagCompare; - mem.image_address_components = 5u; - for (auto &resource : program.info.images) resource.depth_compare = true; - program.info.samplers[0].depth_compare = true; - } else if (rejected == RejectedAccess::Multisampled) { - program.info.images[2].dimension = Dimension::Dim2DMsaaArray; - } else if (rejected == RejectedAccess::Query) { - const auto query = block->PrependNewInst(position, ValueOpcode::ImageQueryDimensions, - {Value(&image), Value(&address)}, memory_flag_bits); - sample.ReplaceUsesWith(Value(&*query), false); - } else { - sample_x.ReplaceUsesWith(Value(0u), false); - sample.Invalidate(); - sampler.Invalidate(); - program.info.samplers.clear(); - program.info.sampled_pairs.clear(); - mem.image_sample_flags = 0u; - mem.image_address_components = 3u; - const bool atomic = rejected == RejectedAccess::Atomic; - for (auto &resource : program.info.images) { - resource.resource_class = ImageResourceClass::Storage; - resource.numeric_class = atomic ? Prospero::TextureNumericClass::Uint - : Prospero::TextureNumericClass::Float; - resource.written = true; - resource.atomic = atomic; - } - if (atomic) { - block->PrependNewInst(position, ValueOpcode::ImageAtomicIAdd32, - {Value(&image), Value(&address), Value(1u), Value(true)}, memory_flag_bits); - } else { - const auto texel = block->PrependNewInst(position, ValueOpcode::CompositeConstructU32x4, - {Value(0u), Value(0u), Value(0u), Value(0u)}); - block->PrependNewInst(position, ValueOpcode::ImageWrite, - {Value(&image), Value(&address), Value(&*texel), Value(true)}, memory_flag_bits); - } - } - program.shader_info_complete = false; - CollectShaderInfo(program, {.compute = &compute}); - ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute}); - }); - } -#endif // Keep an unrelated native root between the null image and appended children, // then select both sides of a cube boundary and the unmapped suffix on the GPU. root.indirect_resources = {0u, 2u, 3u, 4u, 5u, 6u}; @@ -36328,148 +36248,49 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { resource == 4u ? vk::ImageViewType::e2DArray : vk::ImageViewType::e2D, layers)); } const auto native_sampler = vulkan.CreateSampler(test.name); - const auto mip_sampler = vulkan.CreateSampler(test.name, 2u); - constexpr u32 volume_mip_bits = 0x43800001u; - constexpr u32 array_mip_bits = 0x42800001u; - const auto mip_pixels = [](u32 texels, u32 first) { - std::vector pixels(texels * 4u); - for (u32 texel = 0; texel < texels; ++texel) - std::fill_n(pixels.begin() + texel * 4u, 4u, first + texel); - return pixels; - }; - auto volume_texture = vulkan.CreateImageMips( - test.name, 4, 4, vk::Format::eR32G32B32A32Sfloat, vk::ImageUsageFlagBits::eSampled, - {mip_pixels(4u * 4u * 8u, 0x44000001u), mip_pixels(2u * 2u * 4u, volume_mip_bits)}, 4u, - vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e3D, vk::ImageViewType::e3D, 1u, - 0u, 0u, vk::SampleCountFlagBits::e1, vk::Format::eUndefined, 0u, 8u); - auto array_texture = vulkan.CreateImageMips( - test.name, 4, 4, vk::Format::eR32G32B32A32Sfloat, vk::ImageUsageFlagBits::eSampled, - {mip_pixels(4u * 4u * 2u, 0x43000001u), mip_pixels(2u * 2u * 2u, array_mip_bits)}, 4u, - vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2DArray, 2u); - constexpr u32 uint_texel = 0x80000106u; - auto uint_texture = vulkan.CreateImageMips( - test.name, 2, 2, vk::Format::eR32G32B32A32Uint, vk::ImageUsageFlagBits::eSampled, - {std::vector(16u, uint_texel), std::vector(4u, uint_texel + 10u)}, 4u, - vk::ImageLayout::eShaderReadOnlyOptimal, vk::ImageType::e2D, vk::ImageViewType::e2D, 1u); - vk::SamplerCreateInfo integer_sampler_info{}; - integer_sampler_info.addressModeU = integer_sampler_info.addressModeV = - integer_sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; - integer_sampler_info.borderColor = vk::BorderColor::eIntOpaqueWhite; - vk::Sampler integer_sampler; - RequireVk(test.name, "integer sampler", vulkan.Device().createSampler( - &integer_sampler_info, nullptr, &integer_sampler), "vkCreateSampler"); - integer_sampler_info.addressModeU = integer_sampler_info.addressModeV = - integer_sampler_info.addressModeW = vk::SamplerAddressMode::eClampToBorder; - vk::Sampler integer_border_sampler; - RequireVk(test.name, "integer border sampler", vulkan.Device().createSampler( - &integer_sampler_info, nullptr, &integer_border_sampler), "vkCreateSampler"); - const std::array native_samplers{native_sampler, native_sampler, native_sampler, - integer_sampler, integer_sampler, integer_border_sampler}; - enum class Access { Sample, MixedDimensions, Gather, GatherHomogeneous, MixedSample, MixedGather, MixedFetch, Fetch }; + enum class Access { Sample, Gather, GatherHomogeneous, Fetch }; auto *access = &sample; - bool uint_bound = false; - bool volume_bound = false; - for (const auto mode : {Access::Sample, Access::MixedDimensions, Access::Gather, - Access::GatherHomogeneous, Access::MixedSample, - Access::MixedGather, Access::MixedFetch, Access::Fetch}) { - const bool mixed = mode == Access::MixedSample || mode == Access::MixedGather || mode == Access::MixedFetch; - const bool dimensions = mode == Access::MixedDimensions; - const bool fetch = mode == Access::Fetch || mode == Access::MixedFetch; - const bool gather = mode == Access::Gather || mode == Access::GatherHomogeneous || mode == Access::MixedGather; + for (const auto mode : {Access::Sample, Access::Gather, + Access::GatherHomogeneous, Access::Fetch}) { + const bool fetch = mode == Access::Fetch; + const bool gather = mode == Access::Gather || mode == Access::GatherHomogeneous; const bool homogeneous = mode == Access::GatherHomogeneous; - test.name = dimensions ? "IndirectImageArrayCubeVolumeLod" - : mixed ? (fetch ? "IndirectImageMixedFetch" : gather ? "IndirectImageMixedGather" - : "IndirectImageMixedSample") - : fetch ? "IndirectImageFetchContiguousRuns" - : homogeneous ? "IndirectImageGatherHomogeneous" - : gather ? "IndirectImageGatherContiguousRuns" : "IndirectImageContiguousRuns"; - if (volume_bound != dimensions) { - std::swap(textures[6], volume_texture); - std::swap(textures[3], array_texture); - compiled.program.info.images[6].dimension = dimensions - ? ShaderRecompiler::Decoder::ImageDimension::Dim3D - : ShaderRecompiler::Decoder::ImageDimension::Dim2D; - compiled.program.info.images[3].dimension = dimensions - ? ShaderRecompiler::Decoder::ImageDimension::Dim2DArray - : ShaderRecompiler::Decoder::ImageDimension::Dim2D; - volume_bound = dimensions; - } - if (uint_bound != mixed) { - std::swap(textures[6], uint_texture); - compiled.program.info.images[6].numeric_class = mixed ? Prospero::TextureNumericClass::Uint - : Prospero::TextureNumericClass::Float; - uint_bound = mixed; - } - if (mode == Access::MixedSample) { - const auto position = std::ranges::find_if(*block, [&](const Inst &inst) { return &inst == &sampler; }); - const auto sampler_key = block->PrependNewInst(position, ValueOpcode::BitwiseAnd32, - {Value(&key), Value(1u)}); - sampler.SetArg(0, Value(&*sampler_key)); - compiled.program.info.samplers.resize(6); - for (u32 index = 0; index < 6u; ++index) { - auto &resource = compiled.program.info.samplers[index]; - resource.snapshot_index = index % 3u; - resource.force_point_filtering = index >= 3u; - resource.integer_border = index >= 3u; - } - compiled.program.info.samplers[0].indirect_resources = {1u, 2u}; - compiled.program.info.samplers[3].indirect_resources = {4u, 5u}; - compiled.program.info.sampled_pairs.push_back({6u, 3u, 0x10f0u}); - } - const auto opcode = fetch ? ValueOpcode::ImageRead : gather ? ValueOpcode::ImageGatherRaw - : ValueOpcode::ImageSampleRaw; - if (access->GetOpcode() != opcode) { + if (mode == Access::Gather) { + test.name = "IndirectImageGatherContiguousRuns"; const auto position = std::ranges::find_if(*block, [&](const Inst &inst) { return &inst == access; }); - const auto read_at = fetch ? block->PrependNewInst(position, opcode, - {Value(&image), Value(&address), Value(true)}, memory_flag_bits) - : gather ? block->PrependNewInst(position, opcode, - {Value(&image), Value(&sampler), Value(&address), Value(0u), Value(0u), - Value(0u), Value(0u), Value(true)}, memory_flag_bits) - : block->PrependNewInst(position, opcode, - {Value(&image), Value(&sampler), Value(&address), Value(true)}, memory_flag_bits); - auto &read = *read_at; + auto &read = *block->PrependNewInst( + position, ValueOpcode::ImageGatherRaw, + {Value(&image), Value(&sampler), Value(&address), Value(0u), Value(0u), + Value(0u), Value(0u), Value(true)}, memory_flag_bits); access->ReplaceUsesWith(Value(&read), false); access = &read; - } - if (dimensions) { - const auto position = std::ranges::find_if(*block, [&](const Inst &inst) { return &inst == &address; }); - const auto parity = block->PrependNewInst(position, ValueOpcode::BitwiseAnd32, {Value(&key), Value(1u)}); - const auto odd = block->PrependNewInst(position, ValueOpcode::INotEqual32, {Value(&*parity), Value(0u)}); - const auto z = block->PrependNewInst(position, ValueOpcode::SelectU32, - {Value(&*odd), Value(std::bit_cast(0.75f)), Value(std::bit_cast(0.25f))}); - const auto cube_lane = block->PrependNewInst(position, ValueOpcode::IEqual32, {Value(&key), Value(3u)}); - const auto coordinate_z = block->PrependNewInst(position, ValueOpcode::SelectU32, - {Value(&*cube_lane), Value(0u), Value(&*z)}); - address.SetArg(0, Value(std::bit_cast(1.25f))); - address.SetArg(1, Value(std::bit_cast(1.75f))); - address.SetArg(2, Value(&*coordinate_z)); - address.SetArg(3, Value(std::bit_cast(1.0f))); - } else if (gather) { compiled.program.info.images[4].cube = false; auto &gather_memory = compiled.program.memory_info[0]; gather_memory.dmask = 1; gather_memory.image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLevelZero | - (mixed ? 0u : ShaderRecompiler::Decoder::ImageSampleFlagOffset); - // Nonmixed GATHER4_LZ_O places its packed signed offset before the coordinates. - address.SetArg(0, Value(mixed ? std::bit_cast(1.5f) : 0x3fu)); - address.SetArg(1, Value(std::bit_cast(mixed ? 1.5f : 0.75f))); - address.SetArg(2, Value(mixed ? 0u : std::bit_cast(0.25f))); + ShaderRecompiler::Decoder::ImageSampleFlagOffset; + // GATHER4_LZ_O places its packed signed offset before the coordinates. + address.SetArg(0, Value(0x3fu)); + address.SetArg(1, Value(std::bit_cast(0.75f))); + address.SetArg(2, Value(std::bit_cast(0.25f))); address.SetArg(3, Value(0u)); - } else if (mode == Access::MixedSample) { - compiled.program.memory_info[0].dmask = 0xf; - compiled.program.memory_info[0].image_sample_flags = ShaderRecompiler::Decoder::ImageSampleFlagLod; - for (u32 component = 0; component < 4u; ++component) - address.SetArg(component, Value(component < 2u ? std::bit_cast(1.5f) : 0u)); } auto &selected_resources = compiled.program.info.images[0].indirect_resources; selected_resources = homogeneous ? std::vector{0u, 2u, 3u, 5u, 6u} : root.indirect_resources; + if (homogeneous) test.name = "IndirectImageGatherHomogeneous"; compiled.resources.flattened_srt[6] = homogeneous ? 4u : ordinals[5]; - compiled.resources.flattened_srt[7] = mixed || dimensions ? 5u : ordinals[6]; - compiled.resources.flattened_srt[8] = dimensions ? 2u : ordinals[7]; if (fetch) { + test.name = "IndirectImageFetchContiguousRuns"; + const auto position = std::ranges::find_if(*block, [&](const Inst &inst) { + return &inst == access; + }); + auto &read = *block->PrependNewInst( + position, ValueOpcode::ImageRead, + {Value(&image), Value(&address), Value(true)}, memory_flag_bits); + access->ReplaceUsesWith(Value(&read), false); sampler.Invalidate(); compiled.program.info.samplers.clear(); compiled.program.info.sampled_pairs.clear(); @@ -36497,7 +36318,7 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { const auto image_opcode = fetch ? "OpImageFetch" : gather ? "OpImageGather" : "OpImageSampleExplicitLod"; Require(test.name, "native array image runs", tools.Disassemble(compiled.spirv, &text) && - CountText(text, image_opcode) == (homogeneous ? 1u : dimensions ? 5u : mixed ? 4u : 3u) * halves, + CountText(text, image_opcode) == (homogeneous ? 1u : 3u) * halves, "mixed runs expanded into per-image accesses"); if (homogeneous) { Require(test.name, "homogeneous gather array", text.find("OpSwitch") == std::string::npos, @@ -36519,37 +36340,25 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) { Require(test.name, "nonuniform image operands", std::ranges::count_if(image_operands, [&](u32 operand) { return std::ranges::find(decorated, operand) != decorated.end(); - }) == (homogeneous ? 1u : mixed ? (fetch ? 1u : 4u) : 2u) * halves, + }) == (homogeneous ? 1u : 2u) * halves, "a dynamically selected image operand lacks its nonuniform decoration"); } test.initial.assign(wave_size, 0xdeadbeefu); test.expected.assign(wave_size, 0u); for (u32 lane = 0; lane < ordinals.size(); ++lane) { const auto resource = selected_resources[compiled.resources.flattened_srt[1u + lane]]; - test.expected[lane] = dimensions && resource == 6u - ? volume_mip_bits + ((lane & 1u) ? 3u : 1u) * 4u + 3u - : dimensions && resource == 3u ? array_mip_bits + (lane & 1u) * 4u + 3u - : mixed && resource == 6u - ? (fetch ? uint_texel + 10u : (lane & 1u) ? 1u : uint_texel) - : std::bit_cast(float(resource + ((fetch || dimensions) && resource != 0u ? 10u : 0u))); + test.expected[lane] = std::bit_cast(float(resource + (fetch && resource != 0u ? 10u : 0u))); } auto buffer = vulkan.CreateStorageBuffer(test.name, test.initial, test.initial.size()); vulkan.Dispatch(test, compiled, buffer, nullptr, nullptr, nullptr, nullptr, - fetch ? vk::Sampler{} : dimensions ? mip_sampler : native_sampler, textures, - mixed && !fetch ? std::span(native_samplers) : std::span{}); + fetch ? vk::Sampler{} : native_sampler, textures); const auto actual = vulkan.ReadBuffer(test.name, buffer, test.expected.size()); vulkan.DestroyBuffer(&buffer); CompareWords(test, "root, native slots, array runs and mapping bounds", test.expected, actual); } std::printf("[compute] %-32s ok\n", test.name); } - vulkan.Device().destroySampler(integer_sampler); - vulkan.Device().destroySampler(integer_border_sampler); vulkan.Device().destroySampler(native_sampler); - vulkan.Device().destroySampler(mip_sampler); - vulkan.DestroyImage(&uint_texture); - vulkan.DestroyImage(&volume_texture); - vulkan.DestroyImage(&array_texture); for (auto &texture : textures) vulkan.DestroyImage(&texture); } diff --git a/tests/shaderCfgTests.cpp b/tests/shaderCfgTests.cpp index 41378ea89..04e42dd24 100644 --- a/tests/shaderCfgTests.cpp +++ b/tests/shaderCfgTests.cpp @@ -14257,8 +14257,6 @@ void TestBoundedScalarMaterialImageKeys() { uint64_t unavailable = 0; bool clean = true; bool mixed = false; - bool mixed_dimension = false; - bool mixed_conversion = false; bool same_key = false; uint32_t keys = 0; uint32_t images = 0; @@ -14287,13 +14285,10 @@ void TestBoundedScalarMaterialImageKeys() { std::ranges::fill(words, 0u); words[0] = key; const bool integer = key == 5573u || key == 5588u; - words[1] = static_cast(m.mixed_conversion && key == 5573u - ? Prospero::BufferFormat::k11_11_10UInt - : integer && !(m.mixed && key == 5573u) - ? Prospero::BufferFormat::k32UInt : Prospero::BufferFormat::k32Float) << 20u; + words[1] = static_cast(integer && !(m.mixed && key == 5573u) + ? Prospero::BufferFormat::k32UInt : Prospero::BufferFormat::k32Float) << 20u; words[3] = DstSel(4, 5, 6, 7) | - (static_cast(m.mixed_dimension && key == 5573u - ? Prospero::ImageType::kColor2D : Prospero::ImageType::kColor3D) << 28u); + (static_cast(Prospero::ImageType::kColor3D) << 28u); return true; } return false; @@ -14340,22 +14335,8 @@ void TestBoundedScalarMaterialImageKeys() { Check(!MaterializeResources(plan, runtime, snapshot, specialization) && memory.keys == 0, "scalar material byte range escaped V# bounds"); memory = {.mixed = true}; - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.images.size() == 4u && - std::ranges::count_if(specialization.images, [](const auto& image) { - return image.numeric_class == Prospero::TextureNumericClass::Uint; - }) == 1, - "selected material fields lost their mixed float/uint image classes"); - memory = {.mixed_dimension = true}; - Check(MaterializeResources(plan, runtime, snapshot, specialization) && - snapshot.images.size() == 4u && - std::ranges::count_if(specialization.images, [](const auto& image) { - return image.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D; - }) == 1, - "selected material fields lost their per-candidate image dimensions"); - memory = {.mixed_conversion = true}; Check(!MaterializeResources(plan, runtime, snapshot, specialization), - "selected material fields bypassed image conversion compatibility"); + "selected material fields bypassed image compatibility"); memory = {}; Check(MaterializeResources(plan, runtime, snapshot, specialization), "native scalar material image selection did not recover after rejected candidates");