mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-24 04:46:56 +02:00
shader: use native narrow storage for byte-aligned PS5 buffers
Co-authored-by: MehmetCambaz <mcambaz18@gmail.com>
This commit is contained in:
nmzik
@@ -130,7 +130,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
|
||||
const auto aligned_offset = Common::AlignDown(offset, alignment);
|
||||
const auto adjustment = offset - aligned_offset;
|
||||
const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
|
||||
if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
|
||||
if (adjustment >= 256 || size > max_range - adjustment) {
|
||||
EXIT("storage buffer offset adjustment is unsupported\n");
|
||||
}
|
||||
buffer_offset = static_cast<uint32_t>(adjustment);
|
||||
|
||||
@@ -47,6 +47,12 @@ struct VulkanExtensions {
|
||||
std::vector<vk::LayerProperties> available_layers;
|
||||
};
|
||||
|
||||
vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
|
||||
vk::PhysicalDeviceVulkan11Features features {};
|
||||
features.storageBuffer16BitAccess = VK_TRUE;
|
||||
return features;
|
||||
}
|
||||
|
||||
vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
|
||||
vk::PhysicalDeviceVulkan12Features features {};
|
||||
features.samplerMirrorClampToEdge = VK_TRUE;
|
||||
@@ -55,6 +61,7 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
|
||||
features.shaderOutputViewportIndex = VK_TRUE;
|
||||
features.bufferDeviceAddress = VK_TRUE;
|
||||
features.shaderBufferInt64Atomics = VK_TRUE;
|
||||
features.storageBuffer8BitAccess = VK_TRUE;
|
||||
features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
|
||||
return features;
|
||||
}
|
||||
@@ -207,6 +214,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
depth_clip_control.pNext = &depth_clip_enable;
|
||||
|
||||
vk::PhysicalDeviceVulkan12Features features12 {};
|
||||
vk::PhysicalDeviceVulkan11Features features11 {};
|
||||
#if defined(__APPLE__)
|
||||
features12.pNext = &depth_clip_control;
|
||||
#else
|
||||
@@ -214,6 +222,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
fragment_barycentric.pNext = &depth_clip_control;
|
||||
features12.pNext = &fragment_barycentric;
|
||||
#endif
|
||||
features11.pNext = features12.pNext;
|
||||
features12.pNext = &features11;
|
||||
features13.pNext = &features12;
|
||||
device_features2.pNext = &features13;
|
||||
|
||||
@@ -257,6 +267,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
|
||||
check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge",
|
||||
required_features12.samplerMirrorClampToEdge);
|
||||
check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
|
||||
WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
|
||||
check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
|
||||
required_features12.storageBuffer8BitAccess);
|
||||
check_feature(features12.timelineSemaphore, "timelineSemaphore",
|
||||
required_features12.timelineSemaphore);
|
||||
check_feature(features12.shaderOutputLayer, "shaderOutputLayer",
|
||||
@@ -482,6 +496,9 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
workgroup_layout.pNext = &depth_clip_control;
|
||||
features12.pNext = workgroup_layout_extension ? static_cast<void*>(&workgroup_layout)
|
||||
: static_cast<void*>(&depth_clip_control);
|
||||
auto features11 = WindowContext::RequiredVulkan11Features();
|
||||
features11.pNext = features12.pNext;
|
||||
features12.pNext = &features11;
|
||||
if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) {
|
||||
Log::WriteToConsoleAndLog(fmt::format(
|
||||
"WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, "
|
||||
|
||||
@@ -34,6 +34,7 @@ struct WindowContext {
|
||||
~WindowContext();
|
||||
KYTY_CLASS_NO_COPY(WindowContext);
|
||||
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan11Features RequiredVulkan11Features() noexcept;
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan12Features RequiredVulkan12Features() noexcept;
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
|
||||
[[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept;
|
||||
|
||||
@@ -231,6 +231,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
|
||||
if (memory.resource >= program.info.buffers.size()) {
|
||||
Fail(program, "buffer operation has invalid resource metadata");
|
||||
}
|
||||
const auto bits = StorageBufferElementBits(program, memory);
|
||||
requirements.buffer_u8 |= bits == 8u;
|
||||
requirements.buffer_u16 |= bits == 16u;
|
||||
if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
|
||||
if (program.stage != ShaderType::Compute) {
|
||||
Fail(program, "buffer ADD_TID is only valid for compute shaders");
|
||||
|
||||
@@ -70,6 +70,8 @@ struct SpirvRequirements {
|
||||
bool function_scratch = false;
|
||||
bool pixel_valid_mask = false;
|
||||
bool buffer_int64_atomics = false;
|
||||
bool buffer_u8 = false;
|
||||
bool buffer_u16 = false;
|
||||
bool shared_int64_atomics = false;
|
||||
bool coherent_buffers = false;
|
||||
bool float64 = false;
|
||||
@@ -108,6 +110,8 @@ struct EmitterState {
|
||||
uint32_t lane_count = 1;
|
||||
uint32_t lane_half = 0;
|
||||
uint32_t storage_buffer_variable = 0;
|
||||
uint32_t storage_buffer_u8_variable = 0;
|
||||
uint32_t storage_buffer_u16_variable = 0;
|
||||
uint32_t storage_buffer_u64_variable = 0;
|
||||
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
|
||||
uint32_t bda_pagetable_variable = 0;
|
||||
@@ -168,10 +172,9 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
|
||||
uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
|
||||
uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
|
||||
uint32_t TypeFunction(EmitterState& state);
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits);
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits = 32);
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32);
|
||||
uint32_t TypePhysicalU32Pointer(EmitterState& state);
|
||||
uint32_t TypePushConstantElementPointer(EmitterState& state);
|
||||
uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
|
||||
@@ -395,6 +398,7 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
|
||||
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
|
||||
|
||||
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
|
||||
uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem);
|
||||
|
||||
void EmitMemoryOffsets(EmitterState& state);
|
||||
|
||||
@@ -405,9 +409,8 @@ struct MemoryResourceAccess {
|
||||
IR::ResourceKind kind = IR::ResourceKind::None;
|
||||
uint32_t object_pointer = 0;
|
||||
uint32_t length = 0;
|
||||
uint32_t index_offset = 0;
|
||||
uint32_t byte_offset = 0;
|
||||
bool add_index_offset = false;
|
||||
uint32_t element_bits = 32;
|
||||
spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone;
|
||||
};
|
||||
|
||||
@@ -419,9 +422,6 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& st
|
||||
const IR::MemoryInfo& mem,
|
||||
uint32_t variable, uint32_t pointer_type);
|
||||
|
||||
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
|
||||
uint32_t raw_index);
|
||||
|
||||
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
|
||||
uint32_t index);
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <utility>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
@@ -234,7 +235,15 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
|
||||
|
||||
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
if (mem.kind == IR::ResourceKind::Buffer) {
|
||||
return BufferByteAddress(ctx, inst, mem);
|
||||
const auto resource = ResourceForDescriptor(ctx.state, IR::DescriptorBindingKind::Buffers,
|
||||
mem.resource);
|
||||
const auto prefix = ctx.state.memory_byte_offsets[resource];
|
||||
const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state),
|
||||
BufferByteAddress(ctx, inst, mem), prefix);
|
||||
// The host binding prefix must not wrap an out-of-range guest address into the buffer.
|
||||
return Select(ctx.state, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), address, prefix),
|
||||
address, ConstantU32(ctx.state, UINT32_MAX));
|
||||
}
|
||||
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
|
||||
if (mem.offset == 0u) {
|
||||
@@ -254,18 +263,6 @@ uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
|
||||
ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2));
|
||||
}
|
||||
|
||||
struct PreparedMemoryElement {
|
||||
MemoryResourceAccess resource;
|
||||
uint32_t index = 0;
|
||||
};
|
||||
|
||||
PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
uint32_t raw_index) {
|
||||
auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
|
||||
return {.resource = resource, .index = index};
|
||||
}
|
||||
|
||||
uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
|
||||
uint32_t index);
|
||||
|
||||
@@ -274,7 +271,7 @@ uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess&
|
||||
|
||||
uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
return EmitValueOrZeroIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index),
|
||||
[&]() { return LoadWordInBounds(ctx, resource, index); });
|
||||
@@ -290,10 +287,10 @@ uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me
|
||||
uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t bits,
|
||||
bool sign_extend) {
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 2));
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(
|
||||
ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
|
||||
return EmitValueOrZeroIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
return LoadSubwordInBounds(ctx, resource, address, index, bits, sign_extend);
|
||||
@@ -311,15 +308,23 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
|
||||
|
||||
uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
|
||||
uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) {
|
||||
const auto word = LoadWordInBounds(ctx, resource, index);
|
||||
const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto value =
|
||||
Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
|
||||
uint32_t value;
|
||||
if (resource.element_bits < 32u) {
|
||||
const auto loaded = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpLoad, TypeStorageBufferElement(ctx.state, bits), loaded,
|
||||
EmitMemoryElementPointer(ctx.state, resource, index),
|
||||
resource.memory_access);
|
||||
value = Unary(ctx.state, spv::OpUConvert, TypeU32(ctx.state), loaded);
|
||||
} else {
|
||||
const auto word = LoadWordInBounds(ctx, resource, index);
|
||||
const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
|
||||
ConstantU32(ctx.state, 3));
|
||||
value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
|
||||
}
|
||||
if (!sign_extend) return value;
|
||||
const auto left = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), value,
|
||||
ConstantU32(ctx.state, 32u - bits));
|
||||
@@ -408,6 +413,13 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t address, uint32_t index,
|
||||
uint32_t bits, uint32_t data) {
|
||||
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
|
||||
if (resource.element_bits < 32u) {
|
||||
ctx.state.builder.AddFunction(
|
||||
spv::OpStore, pointer,
|
||||
Unary(ctx.state, spv::OpUConvert, TypeStorageBufferElement(ctx.state, bits), data),
|
||||
resource.memory_access);
|
||||
return;
|
||||
}
|
||||
const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 3)),
|
||||
@@ -435,10 +447,10 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
|
||||
void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) {
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 2));
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(
|
||||
ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
|
||||
EmitIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data);
|
||||
@@ -505,7 +517,7 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
|
||||
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t data) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
StoreWordInBounds(ctx, resource, index, data);
|
||||
});
|
||||
@@ -564,10 +576,11 @@ template <typename Fn>
|
||||
uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
|
||||
const IR::MemoryInfo& mem, Fn&& operation) {
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
return EmitValueOrZeroIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
|
||||
return operation(EmitMemoryElementPointer(ctx.state, access.resource, access.index));
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
return operation(EmitMemoryElementPointer(ctx.state, resource, index));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -613,25 +626,15 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
|
||||
PreparedFormattedMemory plan;
|
||||
plan.info = info;
|
||||
plan.resource = resource;
|
||||
auto start = ByteAddress(ctx, inst, mem);
|
||||
plan.in_bounds = ConstantBool(ctx.state, true);
|
||||
if (resource.add_index_offset) {
|
||||
const auto prefixed = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start,
|
||||
resource.byte_offset);
|
||||
plan.in_bounds = Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state),
|
||||
prefixed, start);
|
||||
start = prefixed;
|
||||
}
|
||||
const auto start = ByteAddress(ctx, inst, mem);
|
||||
// Check the whole, unaligned format span before aligning the actual memory address.
|
||||
const auto end = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start,
|
||||
ConstantU32(ctx.state, info.byte_size - 1u));
|
||||
const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end,
|
||||
ConstantU32(ctx.state, 2));
|
||||
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
|
||||
plan.in_bounds = AndCondition(
|
||||
ctx.state, plan.in_bounds,
|
||||
AndCondition(ctx.state,
|
||||
Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start),
|
||||
EmitMemoryElementInBounds(ctx.state, plan.resource, last_index)));
|
||||
ctx.state, Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start),
|
||||
EmitMemoryElementInBounds(ctx.state, plan.resource, last_index));
|
||||
plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start,
|
||||
ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u)));
|
||||
return plan;
|
||||
@@ -645,7 +648,7 @@ std::pair<uint32_t, uint32_t> FormattedComponentAddress(EmitterState& state,
|
||||
const auto address = offset == 0u ? plan.base : Binary(
|
||||
state, spv::OpIAdd, TypeU32(state), plan.base, ConstantU32(state, offset));
|
||||
return {address, Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
|
||||
ConstantU32(state, 2))};
|
||||
ConstantU32(state, std::countr_zero(plan.resource.element_bits / 8u)))};
|
||||
}
|
||||
|
||||
uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
@@ -1031,9 +1034,8 @@ uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co
|
||||
? base
|
||||
: Binary(state, spv::OpIAdd, TypeU32(state), base,
|
||||
ConstantU32(state, component * 4u));
|
||||
const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
|
||||
address, ConstantU32(state, 2));
|
||||
const auto index = EmitMemoryElementIndex(state, resource, raw_index);
|
||||
values[component] = EmitValueOrZeroIfCondition(
|
||||
state, EmitMemoryElementInBounds(state, resource, index),
|
||||
[&]() { return LoadWordInBounds(ctx, resource, index); });
|
||||
@@ -1052,9 +1054,8 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
|
||||
const auto address = component == 0u ? base
|
||||
: Binary(state, spv::OpIAdd, TypeU32(state), base,
|
||||
ConstantU32(state, component * 4u));
|
||||
const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
|
||||
ConstantU32(state, 2));
|
||||
const auto index = EmitMemoryElementIndex(state, resource, raw_index);
|
||||
EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index),
|
||||
[&]() {
|
||||
StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, component + 1u));
|
||||
@@ -1112,7 +1113,7 @@ void DefineGetBdaPointer(EmitterState& state) {
|
||||
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
|
||||
const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64);
|
||||
const auto entry_pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferU64ElementPointer(state),
|
||||
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64),
|
||||
entry_pointer, state.bda_pagetable_variable, ConstantU32(state, 0),
|
||||
page);
|
||||
const auto base = state.builder.AllocateId();
|
||||
@@ -1160,16 +1161,15 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
return EmitValueOrDefaultIfCondition(
|
||||
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
|
||||
const auto resource = PrepareStorageBufferResourceAccess(
|
||||
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
|
||||
const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
|
||||
ByteAddress(ctx, inst, mem), resource.byte_offset);
|
||||
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferPointer(state, 64));
|
||||
const auto byte_address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
|
||||
ConstantU32(state, 3u));
|
||||
return EmitValueOrDefaultIfCondition(
|
||||
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
|
||||
ConstantU64(state, 0), [&]() {
|
||||
const auto pointer = EmitStorageBufferElementPointer(
|
||||
state, resource, index, TypeStorageBufferU64ElementPointer(state));
|
||||
state, resource, index, TypeStorageBufferElementPointer(state, 64));
|
||||
const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
|
||||
EmitAtomicMemoryBarrier(state, mem.kind);
|
||||
return old;
|
||||
@@ -1216,16 +1216,17 @@ void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
EmitIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
ctx.state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable,
|
||||
ctx.Arg(inst, 1));
|
||||
const auto data = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), data,
|
||||
ctx.scratch_u32_variable);
|
||||
AtomicUpdate(
|
||||
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
|
||||
ctx.state, EmitMemoryElementPointer(ctx.state, resource, index),
|
||||
mem.kind, [&](uint32_t old) {
|
||||
const auto old_f =
|
||||
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
|
||||
@@ -1269,10 +1270,9 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
Binary(state, spv::OpIAdd, TypeU32(state), base, size),
|
||||
ConstantU32(state, 0xc000u)));
|
||||
}
|
||||
const auto raw_index =
|
||||
const auto index =
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto index = EmitMemoryElementIndex(state, access, raw_index);
|
||||
const auto exec = ctx.Arg(inst, 1);
|
||||
const auto ballot = ctx.Ballot(inst.Arg(1));
|
||||
const auto low = state.builder.AllocateId();
|
||||
@@ -1343,7 +1343,10 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
|
||||
}
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto element = EmitMemoryElementIndex(state, access, index);
|
||||
// Scalar buffer addressing aligns the base and each offset independently.
|
||||
const auto element = Binary(
|
||||
state, spv::OpIAdd, TypeU32(state), index,
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byte_offset, ConstantU32(state, 2)));
|
||||
const auto condition = EmitMemoryElementInBounds(state, access, element);
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
|
||||
return EmitNative<spv::OpLoad, IR::Type::U32>(
|
||||
|
||||
@@ -92,6 +92,15 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
|
||||
return state.program.info.buffers[mem.resource].descriptor_format;
|
||||
}
|
||||
|
||||
uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) {
|
||||
if (!mem.formatted) return mem.data_bits;
|
||||
const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
|
||||
: program.info.buffers[mem.resource].descriptor_format;
|
||||
const auto info = Format::GetFormatInfo(format);
|
||||
return info.type == Format::ComponentType::Unknown || info.packed_bitfield
|
||||
? 32u : info.component_bits[0];
|
||||
}
|
||||
|
||||
void EmitMemoryOffsets(EmitterState& state) {
|
||||
for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
|
||||
const auto word =
|
||||
@@ -194,24 +203,18 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
|
||||
EXIT("physical address memory must use the BDA emitter\n");
|
||||
case IR::ResourceKind::ScalarBuffer:
|
||||
case IR::ResourceKind::Buffer: {
|
||||
const auto bits = mem.kind == IR::ResourceKind::Buffer
|
||||
? StorageBufferElementBits(state.program, mem) : 32u;
|
||||
const auto variable = bits == 8u ? state.storage_buffer_u8_variable
|
||||
: bits == 16u ? state.storage_buffer_u16_variable
|
||||
: state.storage_buffer_variable;
|
||||
access = PrepareStorageBufferResourceAccess(
|
||||
state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
|
||||
access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
|
||||
ConstantU32(state, 2u));
|
||||
access.add_index_offset = true;
|
||||
state, mem, variable, TypeStorageBufferPointer(state, bits));
|
||||
access.element_bits = bits;
|
||||
return access;
|
||||
}
|
||||
default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
|
||||
}
|
||||
access.length = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
|
||||
access.object_pointer, 0);
|
||||
return access;
|
||||
}
|
||||
|
||||
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
|
||||
uint32_t raw_index) {
|
||||
return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
|
||||
}
|
||||
|
||||
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
|
||||
@@ -240,7 +243,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
|
||||
return pointer;
|
||||
}
|
||||
return EmitStorageBufferElementPointer(state, access, index,
|
||||
TypeStorageBufferElementPointer(state));
|
||||
TypeStorageBufferElementPointer(state, access.element_bits));
|
||||
}
|
||||
|
||||
uint32_t EmitStorageBufferElementPointer(EmitterState& state,
|
||||
|
||||
@@ -123,46 +123,31 @@ uint32_t TypeFunction(EmitterState& state) {
|
||||
return state.function_type;
|
||||
}
|
||||
|
||||
uint32_t StorageRuntimeArrayType(EmitterState& state) {
|
||||
return state.builder.DecoratedType(
|
||||
spv::OpTypeRuntimeArray,
|
||||
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}}, TypeU32(state));
|
||||
uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) {
|
||||
return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state)
|
||||
: state.builder.Type(spv::OpTypeInt, bits, 0);
|
||||
}
|
||||
|
||||
uint32_t StorageBufferType(EmitterState& state) {
|
||||
uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) {
|
||||
return state.builder.DecoratedType(
|
||||
spv::OpTypeRuntimeArray,
|
||||
{{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}},
|
||||
TypeStorageBufferElement(state, bits));
|
||||
}
|
||||
|
||||
uint32_t StorageBufferType(EmitterState& state, uint32_t bits = 32) {
|
||||
return state.builder.DecoratedType(spv::OpTypeStruct,
|
||||
{{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
|
||||
{spv::OpDecorate, {spv::DecorationBlock}}},
|
||||
StorageRuntimeArrayType(state));
|
||||
StorageRuntimeArrayType(state, bits));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state));
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state, bits));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state));
|
||||
}
|
||||
|
||||
uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
|
||||
return state.builder.DecoratedType(
|
||||
spv::OpTypeRuntimeArray,
|
||||
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}}, TypeScalarU64(state));
|
||||
}
|
||||
|
||||
uint32_t StorageBufferU64Type(EmitterState& state) {
|
||||
return state.builder.DecoratedType(spv::OpTypeStruct,
|
||||
{{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
|
||||
{spv::OpDecorate, {spv::DecorationBlock}}},
|
||||
StorageU64RuntimeArrayType(state));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, TypeStorageBufferElement(state, bits));
|
||||
}
|
||||
|
||||
uint32_t TypePhysicalU32Pointer(EmitterState& state) {
|
||||
@@ -244,27 +229,34 @@ void DefineDescriptors(EmitterState& state) {
|
||||
case IR::DescriptorBindingKind::Buffers:
|
||||
state.storage_buffer_variable =
|
||||
Define(ArrayType(StorageBufferType(state)), "buffers");
|
||||
if (state.requirements.buffer_u8) {
|
||||
state.storage_buffer_u8_variable =
|
||||
Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8");
|
||||
}
|
||||
if (state.requirements.buffer_u16) {
|
||||
state.storage_buffer_u16_variable =
|
||||
Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16");
|
||||
}
|
||||
if (state.requirements.buffer_int64_atomics) {
|
||||
state.storage_buffer_u64_variable =
|
||||
Define(ArrayType(StorageBufferU64Type(state)), "buffers_u64");
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
|
||||
spv::DecorationAliased);
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
|
||||
spv::DecorationAliased);
|
||||
Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64");
|
||||
}
|
||||
if (state.requirements.coherent_buffers) {
|
||||
for (const auto variable: {state.storage_buffer_variable, state.storage_buffer_u8_variable,
|
||||
state.storage_buffer_u16_variable, state.storage_buffer_u64_variable}) {
|
||||
if (variable == 0) continue;
|
||||
if (state.storage_buffer_u8_variable != 0 || state.storage_buffer_u16_variable != 0 ||
|
||||
state.storage_buffer_u64_variable != 0) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
|
||||
}
|
||||
// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
|
||||
// must participate in visibility for cache-bypassing polling loads.
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
|
||||
spv::DecorationCoherent);
|
||||
if (state.storage_buffer_u64_variable != 0) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
|
||||
spv::DecorationCoherent);
|
||||
if (state.requirements.coherent_buffers) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case IR::DescriptorBindingKind::BdaPagetable:
|
||||
state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");
|
||||
state.bda_pagetable_variable = Define(StorageBufferType(state, 64), "bda_pagetable");
|
||||
break;
|
||||
case IR::DescriptorBindingKind::FaultBuffer:
|
||||
state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer");
|
||||
@@ -669,6 +661,13 @@ void DefineModule(EmitterState& state) {
|
||||
|
||||
state.builder.RequireCapability(spv::CapabilityShader);
|
||||
state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
|
||||
if (state.requirements.buffer_u8) {
|
||||
state.builder.RequireExtension("SPV_KHR_8bit_storage");
|
||||
state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess);
|
||||
}
|
||||
if (state.requirements.buffer_u16) {
|
||||
state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
|
||||
}
|
||||
if (state.program.info.uses_dma) {
|
||||
state.builder.RequireCapability(spv::CapabilityInt64);
|
||||
state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
|
||||
|
||||
@@ -1311,7 +1311,7 @@ struct TestCase {
|
||||
bool has_user_data = false;
|
||||
u32 image_descriptor_swizzle = DstSel(4, 5, 6, 7);
|
||||
bool compile_only = false;
|
||||
size_t storage_buffer_range_dwords = 0;
|
||||
size_t storage_buffer_range_bytes = 0;
|
||||
std::vector<u32> storage_buffer_offsets;
|
||||
std::vector<BdaMapping> bda_mappings;
|
||||
bool expand_shader_data_storage = false;
|
||||
@@ -1700,7 +1700,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
|
||||
if (resource < test.storage_buffer_offsets.size()) {
|
||||
offset = test.storage_buffer_offsets[resource];
|
||||
}
|
||||
Require(test.name, "shader data", offset % sizeof(u32) == 0 && offset < 256,
|
||||
Require(test.name, "shader data", offset < 256,
|
||||
"storage buffer offset is not representable");
|
||||
packed_user_data[result.program.bindings.memory_offset_dword + i / 4u] |=
|
||||
offset << ((i % 4u) * 8u);
|
||||
@@ -11360,6 +11360,32 @@ public:
|
||||
"descriptor rebind lost its clamped guest range or retained a stale host owner");
|
||||
}
|
||||
|
||||
{
|
||||
const auto buffer_program = make_buffer_program(ShaderType::Compute, {.read = true});
|
||||
constexpr uint64_t buffer_address = base + 0xe0002;
|
||||
ShaderBufferResource descriptor{};
|
||||
descriptor.UpdateAddress48(buffer_address);
|
||||
descriptor.fields[1] |= 6u << 16u;
|
||||
descriptor.fields[2] = 2;
|
||||
descriptor.fields[3] = 0x5204;
|
||||
ShaderRecompiler::IR::ResourceSnapshot snapshot;
|
||||
auto &value = snapshot.buffers.emplace_back();
|
||||
std::copy_n(descriptor.fields, 4, value.dwords.begin());
|
||||
value.dword_count = 4;
|
||||
ShaderStageRuntime runtime{&buffer_program, &snapshot};
|
||||
PreparedBindings bindings;
|
||||
executor.PrepareBindings(runtime, bindings);
|
||||
executor.FindBuffers(bindings);
|
||||
executor.RebindBuffers(bindings);
|
||||
const auto &owner = resources.GetBufferCache().GetBuffer(bindings.buffer_sources[0].id);
|
||||
Require(name, "halfword buffer base and exact tail",
|
||||
bindings.buffers[0].buffer == owner.Handle() &&
|
||||
bindings.buffers[0].offset + 2 == owner.Offset(buffer_address) &&
|
||||
bindings.buffers[0].range == 14 &&
|
||||
bindings.shader_data[buffer_program.bindings.memory_offset_dword] == 2,
|
||||
"native binding lost the byte adjustment or rounded the final halfword range");
|
||||
}
|
||||
|
||||
constexpr auto stencil_format = Prospero::BufferFormat::k8UInt;
|
||||
constexpr auto linear = Prospero::TileMode::kLinear;
|
||||
TileSizeAlign stencil_layout{};
|
||||
@@ -14545,13 +14571,13 @@ public:
|
||||
info.buffer = buffer.buffer;
|
||||
info.offset = 0;
|
||||
info.range = buffer.size;
|
||||
if (test.storage_buffer_range_dwords != 0) {
|
||||
if (test.storage_buffer_range_bytes != 0) {
|
||||
const auto resource = buffers->resources[i];
|
||||
const auto offset = resource < test.storage_buffer_offsets.size()
|
||||
? test.storage_buffer_offsets[resource]
|
||||
: 0u;
|
||||
info.range = static_cast<vk::DeviceSize>(
|
||||
test.storage_buffer_range_dwords * sizeof(u32) + offset);
|
||||
test.storage_buffer_range_bytes + offset);
|
||||
Require(test.name, "dispatch", info.range <= buffer.size,
|
||||
"storage buffer descriptor range exceeds backing buffer");
|
||||
}
|
||||
@@ -17117,10 +17143,12 @@ private:
|
||||
if ((queues[i].queueFlags &
|
||||
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) ==
|
||||
(vk::QueueFlagBits::eCompute | vk::QueueFlagBits::eGraphics)) {
|
||||
vk::PhysicalDeviceVulkan11Features features11{};
|
||||
vk::PhysicalDeviceVulkan12Features features12{};
|
||||
features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
|
||||
vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
|
||||
features12.pNext = &workgroup_layout;
|
||||
features12.pNext = &features11;
|
||||
features11.pNext = &workgroup_layout;
|
||||
vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
|
||||
workgroup_layout.pNext = &image_atomic64;
|
||||
vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR barycentric{};
|
||||
@@ -17132,6 +17160,8 @@ private:
|
||||
physical.getFeatures2(&features);
|
||||
if (barycentric.fragmentShaderBarycentric != true ||
|
||||
features.features.shaderInt64 != true ||
|
||||
features11.storageBuffer16BitAccess != true ||
|
||||
features12.storageBuffer8BitAccess != true ||
|
||||
features12.samplerMirrorClampToEdge != true ||
|
||||
features12.shaderOutputViewportIndex != true ||
|
||||
features12.shaderBufferInt64Atomics != true ||
|
||||
@@ -17157,7 +17187,9 @@ private:
|
||||
|
||||
vk::PhysicalDeviceFeatures available_features{};
|
||||
m_physical_device.getFeatures(&available_features);
|
||||
vk::PhysicalDeviceVulkan11Features available_features11{};
|
||||
vk::PhysicalDeviceVulkan12Features available_features12{};
|
||||
available_features12.pNext = &available_features11;
|
||||
available_features12.sType =
|
||||
vk::StructureType::ePhysicalDeviceVulkan12Features;
|
||||
vk::PhysicalDeviceVulkan13Features available_features13{};
|
||||
@@ -17191,6 +17223,10 @@ private:
|
||||
available_features.shaderImageGatherExtended == true &&
|
||||
available_derivatives.computeDerivativeGroupQuads == true,
|
||||
"image gather or compute derivative quads are not supported");
|
||||
Require("VulkanHarness", "dispatch",
|
||||
available_features11.storageBuffer16BitAccess == true &&
|
||||
available_features12.storageBuffer8BitAccess == true,
|
||||
"8-bit and 16-bit storage buffer access are not supported");
|
||||
Require("VulkanHarness", "dispatch",
|
||||
available_features12.timelineSemaphore == true,
|
||||
"timeline semaphores are not supported");
|
||||
@@ -17240,11 +17276,13 @@ private:
|
||||
device_info.sType = vk::StructureType::eDeviceCreateInfo;
|
||||
device_info.queueCreateInfoCount = 1;
|
||||
device_info.pQueueCreateInfos = &queue_info;
|
||||
auto device_features11 = WindowContext::RequiredVulkan11Features();
|
||||
auto device_features12 = WindowContext::RequiredVulkan12Features();
|
||||
device_features12.shaderSharedInt64Atomics = true;
|
||||
vk::PhysicalDeviceWorkgroupMemoryExplicitLayoutFeaturesKHR workgroup_layout{};
|
||||
workgroup_layout.workgroupMemoryExplicitLayout = true;
|
||||
device_features12.pNext = &workgroup_layout;
|
||||
device_features12.pNext = &device_features11;
|
||||
device_features11.pNext = &workgroup_layout;
|
||||
vk::PhysicalDeviceShaderImageAtomicInt64FeaturesEXT image_atomic64{};
|
||||
image_atomic64.shaderImageInt64Atomics = true;
|
||||
workgroup_layout.pNext = &image_atomic64;
|
||||
@@ -26186,7 +26224,7 @@ TestCase BufferStoreDwordAppliesHostOffset() {
|
||||
|
||||
std::vector<u32> code;
|
||||
AppendVMovLiteral(&code, 0, 0xabcdef01u);
|
||||
code.push_back(EncodeMubuf0(0x1cu, 0, false, false));
|
||||
code.push_back(EncodeMubuf0(0x1cu, 2, false, false));
|
||||
code.push_back(EncodeMubuf1(0, 0, 20));
|
||||
AppendEnd(&code);
|
||||
|
||||
@@ -26194,11 +26232,11 @@ TestCase BufferStoreDwordAppliesHostOffset() {
|
||||
test.name = "BufferStoreDwordAppliesHostOffset";
|
||||
test.code = code;
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0};
|
||||
test.expected = {0x11111111u, 0x22222222u, 0x33333333u, 0xabcdef01u};
|
||||
test.storage_buffer_range_dwords = 1;
|
||||
test.storage_buffer_offsets = {12};
|
||||
test.expected = {0x11111111u, 0xabcdef01u, 0x33333333u, 0};
|
||||
test.storage_buffer_range_bytes = 8;
|
||||
test.storage_buffer_offsets = {2};
|
||||
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
|
||||
test.user_data = MakeStructuredStorageBufferData(4, 1);
|
||||
test.user_data = MakeStructuredStorageBufferData(0, 8);
|
||||
test.has_user_data = true;
|
||||
return test;
|
||||
}
|
||||
@@ -26226,7 +26264,7 @@ TestCase BufferOffsetsUsePackedLaneAndStorageFallback() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0, 0, 0, 0};
|
||||
test.expected = {0x12345678u, 0, 0, 0xabcdef01u};
|
||||
test.storage_buffer_range_dwords = 1;
|
||||
test.storage_buffer_range_bytes = 4;
|
||||
test.storage_buffer_offsets = {0, 12};
|
||||
test.expand_shader_data_storage = true;
|
||||
test.opcodes = {O::S_MOV_B32, O::V_MOV_B32, O::BUFFER_STORE_DWORD,
|
||||
@@ -26269,6 +26307,99 @@ TestCase BufferLoadVariants() {
|
||||
O::BUFFER_STORE_DWORD, O::S_ENDPGM}};
|
||||
}
|
||||
|
||||
TestCase BufferSubwordLoadsAtHostOffset(u32 bytes, u32 component, bool sign) {
|
||||
using O = ShaderOpcode;
|
||||
TestCase test;
|
||||
test.name = bytes == 1 ? (sign ? "BufferSbyteHostOffsetTail" : "BufferUbyteHostOffsetTail")
|
||||
: sign ? "BufferSshortHostOffsetTail"
|
||||
: component == 0 ? "BufferUshortHostOffsetFirst"
|
||||
: component == 2 ? "BufferUshortHostOffsetCarry"
|
||||
: "BufferUshortHostOffsetTail";
|
||||
// The captured stride-six index buffer uses USHORT IDXEN offsets 4, 2, 0.
|
||||
test.code = {EncodeMubuf0((bytes == 1 ? 0x08u : 0x0au) + sign,
|
||||
component, true, false),
|
||||
EncodeMubuf1(1, 0, 0),
|
||||
EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)};
|
||||
AppendEnd(&test.code);
|
||||
test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
|
||||
0xccccccccu, 0xccccccccu, 0xccccccccu};
|
||||
test.expected = test.initial;
|
||||
for (u32 record = 0; record < 2; ++record) {
|
||||
u32 value = 0;
|
||||
std::memcpy(&value, reinterpret_cast<const uint8_t *>(test.initial.data()) +
|
||||
bytes + record * 6 + component, bytes);
|
||||
if (sign && (value & (1u << (bytes * 8u - 1u))) != 0) {
|
||||
value |= UINT32_MAX << (bytes * 8u);
|
||||
}
|
||||
test.expected[4 + record] = value;
|
||||
}
|
||||
test.expected[6] = 0;
|
||||
test.storage_buffer_range_bytes = 12;
|
||||
test.storage_buffer_offsets = {bytes, 16};
|
||||
test.user_data = MakeStructuredStorageBufferData(6, 2);
|
||||
test.user_data[0] = 0x1000u + bytes;
|
||||
test.user_data[48] = 0x2000u;
|
||||
test.user_data[49] = 4u << 16u;
|
||||
test.user_data[50] = 3;
|
||||
test.user_data[51] = 0x5204;
|
||||
test.has_user_data = true;
|
||||
test.compute_info.thread_ids_num = 1;
|
||||
test.compute_info.threads_num[0] = 3;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
test.has_compute_info = true;
|
||||
test.opcodes = {bytes == 1 ? (sign ? O::BUFFER_LOAD_SBYTE : O::BUFFER_LOAD_UBYTE)
|
||||
: (sign ? O::BUFFER_LOAD_SSHORT : O::BUFFER_LOAD_USHORT),
|
||||
O::BUFFER_STORE_DWORD, O::S_ENDPGM};
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase BufferSubwordHostOffsetOverflow() {
|
||||
auto test = BufferSubwordLoadsAtHostOffset(2, 0, false);
|
||||
test.name = "BufferSubwordHostOffsetOverflow";
|
||||
test.code.clear();
|
||||
AppendVMovLiteral(&test.code, 20, UINT32_MAX);
|
||||
test.code.insert(test.code.end(),
|
||||
{EncodeMubuf0(0x0au), EncodeMubuf1(1, 0, 20),
|
||||
EncodeMubuf0(0x1cu, 0, true, false), EncodeMubuf1(1, 12, 0)});
|
||||
AppendVMovLiteral(&test.code, 1, 0xbeef);
|
||||
test.code.insert(test.code.end(), {EncodeMubuf0(0x1au), EncodeMubuf1(1, 0, 20)});
|
||||
AppendEnd(&test.code);
|
||||
test.expected[4] = test.expected[5] = 0;
|
||||
test.opcodes.push_back(ShaderOpcode::V_MOV_B32);
|
||||
test.opcodes.push_back(ShaderOpcode::BUFFER_STORE_SHORT);
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase BufferSubwordStoresAtHostOffset(u32 bytes) {
|
||||
using O = ShaderOpcode;
|
||||
TestCase test;
|
||||
test.name = bytes == 1 ? "BufferByteStoreHostOffsetTail" : "BufferShortStoreHostOffsetTail";
|
||||
AppendVMovLiteral(&test.code, 1, bytes == 1 ? 0xa5u : 0xcafeu);
|
||||
test.code.push_back(EncodeMubuf0(bytes == 1 ? 0x18u : 0x1au,
|
||||
6u - bytes, true, false));
|
||||
test.code.push_back(EncodeMubuf1(1, 0, 0));
|
||||
AppendEnd(&test.code);
|
||||
test.initial = {0x800112abu, 0xfedc2345u, 0x80007fffu, 0xfaceffffu,
|
||||
0x13579bdfu, 0x2468ace0u};
|
||||
test.expected = bytes == 1
|
||||
? std::vector<u32>{0x800112abu, 0xfea52345u, 0x80007fffu, 0xfaceffa5u,
|
||||
0x13579bdfu, 0x2468ace0u}
|
||||
: std::vector<u32>{0x800112abu, 0xcafe2345u, 0x80007fffu, 0xfacecafeu,
|
||||
0x13579bdfu, 0x2468ace0u};
|
||||
test.storage_buffer_range_bytes = 12;
|
||||
test.storage_buffer_offsets = {bytes};
|
||||
test.user_data = MakeStructuredStorageBufferData(6, 2);
|
||||
test.user_data[0] = 0x1000u + bytes;
|
||||
test.has_user_data = true;
|
||||
test.compute_info.thread_ids_num = 1;
|
||||
test.compute_info.threads_num[0] = 3;
|
||||
test.compute_info.threads_num[1] = test.compute_info.threads_num[2] = 1;
|
||||
test.has_compute_info = true;
|
||||
test.opcodes = {O::V_MOV_B32, bytes == 1 ? O::BUFFER_STORE_BYTE : O::BUFFER_STORE_SHORT,
|
||||
O::S_ENDPGM};
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase BufferLoadDwordx4SnapshotsOverlappingAddress() {
|
||||
using O = ShaderOpcode;
|
||||
|
||||
@@ -26363,7 +26494,7 @@ TestCase BufferLoadDwordx4ZeroesOnlyOutOfBoundsTail() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = {0x33333333u, 0x44444444u, 0u, 0u};
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.opcodes = {O::V_MOV_B32, O::BUFFER_LOAD_DWORDX4, O::BUFFER_STORE_DWORD,
|
||||
O::S_ENDPGM};
|
||||
return test;
|
||||
@@ -26574,7 +26705,7 @@ TestCase BufferStoreDwordx4DropsOnlyOutOfBoundsTail() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = {0x11111111u, 0x22222222u, 0xaaaaaaaau, 0xbbbbbbbbu};
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_DWORDX4, O::S_ENDPGM};
|
||||
return test;
|
||||
}
|
||||
@@ -26595,7 +26726,7 @@ TestCase BufferLoadFormatXyzwRejectsPartialRecord() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x3f800000u, 0x40000000u, 0x40400000u, 0x40800000u};
|
||||
test.expected = {0u, 0u, 0u, 0u};
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.user_data = MakeStructuredStorageBufferData(
|
||||
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
|
||||
test.has_user_data = true;
|
||||
@@ -26622,7 +26753,7 @@ TestCase BufferStoreFormatXyzwDropsPartialRecord() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = test.initial;
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.user_data = MakeStructuredStorageBufferData(
|
||||
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
|
||||
test.has_user_data = true;
|
||||
@@ -26648,7 +26779,7 @@ TestCase BufferLoadFormatXRejectsPartialRecord() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.user_data = MakeStructuredStorageBufferData(
|
||||
0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
|
||||
test.has_user_data = true;
|
||||
@@ -26672,7 +26803,7 @@ TestCase BufferStoreFormatXRejectsPartialRecord() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = test.initial;
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.user_data = MakeStructuredStorageBufferData(
|
||||
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
|
||||
test.has_user_data = true;
|
||||
@@ -26695,7 +26826,7 @@ TestCase BufferLoadFormatXyRejectsPartialRecord() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = {0u, 0u, 0x33333333u, 0x44444444u};
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.user_data = MakeStructuredStorageBufferData(
|
||||
0, 16, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
|
||||
test.has_user_data = true;
|
||||
@@ -26720,7 +26851,7 @@ TestCase BufferStoreFormatXyRejectsPartialRecord() {
|
||||
test.code = std::move(code);
|
||||
test.initial = {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u};
|
||||
test.expected = test.initial;
|
||||
test.storage_buffer_range_dwords = 4;
|
||||
test.storage_buffer_range_bytes = 16;
|
||||
test.user_data = MakeStructuredStorageBufferData(
|
||||
0, 4, false, BufferFormat(Prospero::BufferFormat::k32_32_32_32Float));
|
||||
test.has_user_data = true;
|
||||
@@ -27030,7 +27161,7 @@ TestCase BufferStoreFormatXResource16UintPreservesAdjacentLanes() {
|
||||
test.has_compute_info = true;
|
||||
test.user_data = MakeStructuredStorageBufferData(2, 64, false, 11);
|
||||
test.has_user_data = true;
|
||||
test.required_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"};
|
||||
test.forbidden_spirv = {"OpAtomicLoad", "OpAtomicCompareExchange"};
|
||||
return test;
|
||||
}
|
||||
|
||||
@@ -27383,7 +27514,7 @@ TestCase BufferStoreFormatXDropsOutOfRangeRecord() {
|
||||
test.code = code;
|
||||
test.initial = {0x11111111u, 0x22222222u};
|
||||
test.expected = {0x11111111u, 0x22222222u};
|
||||
test.storage_buffer_range_dwords = 1;
|
||||
test.storage_buffer_range_bytes = 4;
|
||||
test.opcodes = {O::V_MOV_B32, O::BUFFER_STORE_FORMAT_X, O::S_ENDPGM};
|
||||
test.user_data = MakeStructuredStorageBufferData(4, 1, false, 20);
|
||||
test.has_user_data = true;
|
||||
@@ -31534,7 +31665,7 @@ void CheckIndirectBufferStore(VulkanHarness &vulkan) {
|
||||
TestCase test;
|
||||
test.name = "IndirectBufferStore";
|
||||
test.initial.assign(64, 0xdeadbeefu);
|
||||
test.storage_buffer_range_dwords = 16;
|
||||
test.storage_buffer_range_bytes = 64;
|
||||
test.storage_buffer_offsets = {12, 108};
|
||||
test.required_spirv = {"OpSwitch"};
|
||||
test.forbidden_spirv = {"PhysicalStorageBuffer"};
|
||||
@@ -33303,6 +33434,15 @@ std::vector<TestCase> MakeCases() {
|
||||
AddCase(BufferStoreDwordAppliesHostOffset);
|
||||
AddCase(BufferOffsetsUsePackedLaneAndStorageFallback);
|
||||
AddCase(BufferLoadVariants);
|
||||
for (const u32 component : {4u, 2u, 0u}) {
|
||||
cases.push_back(BufferSubwordLoadsAtHostOffset(2, component, false));
|
||||
}
|
||||
cases.push_back(BufferSubwordLoadsAtHostOffset(2, 4, true));
|
||||
cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, false));
|
||||
cases.push_back(BufferSubwordLoadsAtHostOffset(1, 5, true));
|
||||
AddCase(BufferSubwordHostOffsetOverflow);
|
||||
cases.push_back(BufferSubwordStoresAtHostOffset(2));
|
||||
cases.push_back(BufferSubwordStoresAtHostOffset(1));
|
||||
AddCase(BufferLoadDwordx2SnapshotsOverlappingAddress);
|
||||
AddCase(BufferLoadDwordx3SnapshotsOverlappingAddress);
|
||||
AddCase(BufferLoadDwordx4SnapshotsOverlappingAddress);
|
||||
|
||||
Reference in New Issue
Block a user