mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-20 19:07:01 +02:00
shader: use native narrow storage for byte-aligned PS5 buffers
Co-authored-by: MehmetCambaz <mcambaz18@gmail.com>
This commit is contained in:
nmzik
@@ -130,7 +130,7 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
|
||||
const auto aligned_offset = Common::AlignDown(offset, alignment);
|
||||
const auto adjustment = offset - aligned_offset;
|
||||
const auto max_range = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
|
||||
if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
|
||||
if (adjustment >= 256 || size > max_range - adjustment) {
|
||||
EXIT("storage buffer offset adjustment is unsupported\n");
|
||||
}
|
||||
buffer_offset = static_cast<uint32_t>(adjustment);
|
||||
|
||||
@@ -47,6 +47,12 @@ struct VulkanExtensions {
|
||||
std::vector<vk::LayerProperties> available_layers;
|
||||
};
|
||||
|
||||
vk::PhysicalDeviceVulkan11Features WindowContext::RequiredVulkan11Features() noexcept {
|
||||
vk::PhysicalDeviceVulkan11Features features {};
|
||||
features.storageBuffer16BitAccess = VK_TRUE;
|
||||
return features;
|
||||
}
|
||||
|
||||
vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noexcept {
|
||||
vk::PhysicalDeviceVulkan12Features features {};
|
||||
features.samplerMirrorClampToEdge = VK_TRUE;
|
||||
@@ -55,6 +61,7 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
|
||||
features.shaderOutputViewportIndex = VK_TRUE;
|
||||
features.bufferDeviceAddress = VK_TRUE;
|
||||
features.shaderBufferInt64Atomics = VK_TRUE;
|
||||
features.storageBuffer8BitAccess = VK_TRUE;
|
||||
features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
|
||||
return features;
|
||||
}
|
||||
@@ -207,6 +214,7 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
depth_clip_control.pNext = &depth_clip_enable;
|
||||
|
||||
vk::PhysicalDeviceVulkan12Features features12 {};
|
||||
vk::PhysicalDeviceVulkan11Features features11 {};
|
||||
#if defined(__APPLE__)
|
||||
features12.pNext = &depth_clip_control;
|
||||
#else
|
||||
@@ -214,6 +222,8 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
fragment_barycentric.pNext = &depth_clip_control;
|
||||
features12.pNext = &fragment_barycentric;
|
||||
#endif
|
||||
features11.pNext = features12.pNext;
|
||||
features12.pNext = &features11;
|
||||
features13.pNext = &features12;
|
||||
device_features2.pNext = &features13;
|
||||
|
||||
@@ -257,6 +267,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
|
||||
check_feature(features12.samplerMirrorClampToEdge, "samplerMirrorClampToEdge",
|
||||
required_features12.samplerMirrorClampToEdge);
|
||||
check_feature(features11.storageBuffer16BitAccess, "storageBuffer16BitAccess",
|
||||
WindowContext::RequiredVulkan11Features().storageBuffer16BitAccess);
|
||||
check_feature(features12.storageBuffer8BitAccess, "storageBuffer8BitAccess",
|
||||
required_features12.storageBuffer8BitAccess);
|
||||
check_feature(features12.timelineSemaphore, "timelineSemaphore",
|
||||
required_features12.timelineSemaphore);
|
||||
check_feature(features12.shaderOutputLayer, "shaderOutputLayer",
|
||||
@@ -482,6 +496,9 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
workgroup_layout.pNext = &depth_clip_control;
|
||||
features12.pNext = workgroup_layout_extension ? static_cast<void*>(&workgroup_layout)
|
||||
: static_cast<void*>(&depth_clip_control);
|
||||
auto features11 = WindowContext::RequiredVulkan11Features();
|
||||
features11.pNext = features12.pNext;
|
||||
features12.pNext = &features11;
|
||||
if (!features12.shaderSharedInt64Atomics || !workgroup_layout.workgroupMemoryExplicitLayout) {
|
||||
Log::WriteToConsoleAndLog(fmt::format(
|
||||
"WARNING: Native 64-bit LDS atomics are unavailable: shaderSharedInt64Atomics={}, "
|
||||
|
||||
@@ -34,6 +34,7 @@ struct WindowContext {
|
||||
~WindowContext();
|
||||
KYTY_CLASS_NO_COPY(WindowContext);
|
||||
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan11Features RequiredVulkan11Features() noexcept;
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan12Features RequiredVulkan12Features() noexcept;
|
||||
[[nodiscard]] static vk::PhysicalDeviceVulkan13Features RequiredVulkan13Features() noexcept;
|
||||
[[nodiscard]] static uint32_t InitialWindowFlags(bool fullscreen) noexcept;
|
||||
|
||||
@@ -231,6 +231,9 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
|
||||
if (memory.resource >= program.info.buffers.size()) {
|
||||
Fail(program, "buffer operation has invalid resource metadata");
|
||||
}
|
||||
const auto bits = StorageBufferElementBits(program, memory);
|
||||
requirements.buffer_u8 |= bits == 8u;
|
||||
requirements.buffer_u16 |= bits == 16u;
|
||||
if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
|
||||
if (program.stage != ShaderType::Compute) {
|
||||
Fail(program, "buffer ADD_TID is only valid for compute shaders");
|
||||
|
||||
@@ -70,6 +70,8 @@ struct SpirvRequirements {
|
||||
bool function_scratch = false;
|
||||
bool pixel_valid_mask = false;
|
||||
bool buffer_int64_atomics = false;
|
||||
bool buffer_u8 = false;
|
||||
bool buffer_u16 = false;
|
||||
bool shared_int64_atomics = false;
|
||||
bool coherent_buffers = false;
|
||||
bool float64 = false;
|
||||
@@ -108,6 +110,8 @@ struct EmitterState {
|
||||
uint32_t lane_count = 1;
|
||||
uint32_t lane_half = 0;
|
||||
uint32_t storage_buffer_variable = 0;
|
||||
uint32_t storage_buffer_u8_variable = 0;
|
||||
uint32_t storage_buffer_u16_variable = 0;
|
||||
uint32_t storage_buffer_u64_variable = 0;
|
||||
std::array<uint32_t, IR::ShaderInfo::MaxBuffers> memory_byte_offsets {};
|
||||
uint32_t bda_pagetable_variable = 0;
|
||||
@@ -168,10 +172,9 @@ uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
|
||||
uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
|
||||
uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
|
||||
uint32_t TypeFunction(EmitterState& state);
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
|
||||
uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits);
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits = 32);
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits = 32);
|
||||
uint32_t TypePhysicalU32Pointer(EmitterState& state);
|
||||
uint32_t TypePushConstantElementPointer(EmitterState& state);
|
||||
uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
|
||||
@@ -395,6 +398,7 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
|
||||
uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
|
||||
|
||||
Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
|
||||
uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem);
|
||||
|
||||
void EmitMemoryOffsets(EmitterState& state);
|
||||
|
||||
@@ -405,9 +409,8 @@ struct MemoryResourceAccess {
|
||||
IR::ResourceKind kind = IR::ResourceKind::None;
|
||||
uint32_t object_pointer = 0;
|
||||
uint32_t length = 0;
|
||||
uint32_t index_offset = 0;
|
||||
uint32_t byte_offset = 0;
|
||||
bool add_index_offset = false;
|
||||
uint32_t element_bits = 32;
|
||||
spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone;
|
||||
};
|
||||
|
||||
@@ -419,9 +422,6 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& st
|
||||
const IR::MemoryInfo& mem,
|
||||
uint32_t variable, uint32_t pointer_type);
|
||||
|
||||
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
|
||||
uint32_t raw_index);
|
||||
|
||||
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
|
||||
uint32_t index);
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <bit>
|
||||
#include <utility>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
@@ -234,7 +235,15 @@ uint32_t LoadBda(ValueEmitContext& ctx, uint32_t address, uint32_t active, uint3
|
||||
|
||||
uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
if (mem.kind == IR::ResourceKind::Buffer) {
|
||||
return BufferByteAddress(ctx, inst, mem);
|
||||
const auto resource = ResourceForDescriptor(ctx.state, IR::DescriptorBindingKind::Buffers,
|
||||
mem.resource);
|
||||
const auto prefix = ctx.state.memory_byte_offsets[resource];
|
||||
const auto address = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state),
|
||||
BufferByteAddress(ctx, inst, mem), prefix);
|
||||
// The host binding prefix must not wrap an out-of-range guest address into the buffer.
|
||||
return Select(ctx.state, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), address, prefix),
|
||||
address, ConstantU32(ctx.state, UINT32_MAX));
|
||||
}
|
||||
if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
|
||||
if (mem.offset == 0u) {
|
||||
@@ -254,18 +263,6 @@ uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
|
||||
ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2));
|
||||
}
|
||||
|
||||
struct PreparedMemoryElement {
|
||||
MemoryResourceAccess resource;
|
||||
uint32_t index = 0;
|
||||
};
|
||||
|
||||
PreparedMemoryElement PrepareMemoryElement(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
uint32_t raw_index) {
|
||||
auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
|
||||
return {.resource = resource, .index = index};
|
||||
}
|
||||
|
||||
uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
|
||||
uint32_t index);
|
||||
|
||||
@@ -274,7 +271,7 @@ uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess&
|
||||
|
||||
uint32_t LoadWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
return EmitValueOrZeroIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index),
|
||||
[&]() { return LoadWordInBounds(ctx, resource, index); });
|
||||
@@ -290,10 +287,10 @@ uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me
|
||||
uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t bits,
|
||||
bool sign_extend) {
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 2));
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(
|
||||
ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
|
||||
return EmitValueOrZeroIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
return LoadSubwordInBounds(ctx, resource, address, index, bits, sign_extend);
|
||||
@@ -311,15 +308,23 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
|
||||
|
||||
uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
|
||||
uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) {
|
||||
const auto word = LoadWordInBounds(ctx, resource, index);
|
||||
const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto value =
|
||||
Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
|
||||
uint32_t value;
|
||||
if (resource.element_bits < 32u) {
|
||||
const auto loaded = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpLoad, TypeStorageBufferElement(ctx.state, bits), loaded,
|
||||
EmitMemoryElementPointer(ctx.state, resource, index),
|
||||
resource.memory_access);
|
||||
value = Unary(ctx.state, spv::OpUConvert, TypeU32(ctx.state), loaded);
|
||||
} else {
|
||||
const auto word = LoadWordInBounds(ctx, resource, index);
|
||||
const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 3));
|
||||
const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
|
||||
ConstantU32(ctx.state, 3));
|
||||
value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
|
||||
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
|
||||
}
|
||||
if (!sign_extend) return value;
|
||||
const auto left = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), value,
|
||||
ConstantU32(ctx.state, 32u - bits));
|
||||
@@ -408,6 +413,13 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t address, uint32_t index,
|
||||
uint32_t bits, uint32_t data) {
|
||||
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
|
||||
if (resource.element_bits < 32u) {
|
||||
ctx.state.builder.AddFunction(
|
||||
spv::OpStore, pointer,
|
||||
Unary(ctx.state, spv::OpUConvert, TypeStorageBufferElement(ctx.state, bits), data),
|
||||
resource.memory_access);
|
||||
return;
|
||||
}
|
||||
const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 3)),
|
||||
@@ -435,10 +447,10 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
|
||||
void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) {
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, 2));
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
|
||||
const auto address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(
|
||||
ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
|
||||
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
|
||||
EmitIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data);
|
||||
@@ -505,7 +517,7 @@ void StoreLocalFlat(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
|
||||
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t data) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
StoreWordInBounds(ctx, resource, index, data);
|
||||
});
|
||||
@@ -564,10 +576,11 @@ template <typename Fn>
|
||||
uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
|
||||
const IR::MemoryInfo& mem, Fn&& operation) {
|
||||
return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
return EmitValueOrZeroIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
|
||||
return operation(EmitMemoryElementPointer(ctx.state, access.resource, access.index));
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
return operation(EmitMemoryElementPointer(ctx.state, resource, index));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -613,25 +626,15 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
|
||||
PreparedFormattedMemory plan;
|
||||
plan.info = info;
|
||||
plan.resource = resource;
|
||||
auto start = ByteAddress(ctx, inst, mem);
|
||||
plan.in_bounds = ConstantBool(ctx.state, true);
|
||||
if (resource.add_index_offset) {
|
||||
const auto prefixed = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start,
|
||||
resource.byte_offset);
|
||||
plan.in_bounds = Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state),
|
||||
prefixed, start);
|
||||
start = prefixed;
|
||||
}
|
||||
const auto start = ByteAddress(ctx, inst, mem);
|
||||
// Check the whole, unaligned format span before aligning the actual memory address.
|
||||
const auto end = Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), start,
|
||||
ConstantU32(ctx.state, info.byte_size - 1u));
|
||||
const auto last_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), end,
|
||||
ConstantU32(ctx.state, 2));
|
||||
ConstantU32(ctx.state, std::countr_zero(resource.element_bits / 8u)));
|
||||
plan.in_bounds = AndCondition(
|
||||
ctx.state, plan.in_bounds,
|
||||
AndCondition(ctx.state,
|
||||
Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start),
|
||||
EmitMemoryElementInBounds(ctx.state, plan.resource, last_index)));
|
||||
ctx.state, Binary(ctx.state, spv::OpUGreaterThanEqual, TypeBool(ctx.state), end, start),
|
||||
EmitMemoryElementInBounds(ctx.state, plan.resource, last_index));
|
||||
plan.base = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), start,
|
||||
ConstantU32(ctx.state, ~(std::min(4u, info.byte_size) - 1u)));
|
||||
return plan;
|
||||
@@ -645,7 +648,7 @@ std::pair<uint32_t, uint32_t> FormattedComponentAddress(EmitterState& state,
|
||||
const auto address = offset == 0u ? plan.base : Binary(
|
||||
state, spv::OpIAdd, TypeU32(state), plan.base, ConstantU32(state, offset));
|
||||
return {address, Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
|
||||
ConstantU32(state, 2))};
|
||||
ConstantU32(state, std::countr_zero(plan.resource.element_bits / 8u)))};
|
||||
}
|
||||
|
||||
uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
@@ -1031,9 +1034,8 @@ uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co
|
||||
? base
|
||||
: Binary(state, spv::OpIAdd, TypeU32(state), base,
|
||||
ConstantU32(state, component * 4u));
|
||||
const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
|
||||
address, ConstantU32(state, 2));
|
||||
const auto index = EmitMemoryElementIndex(state, resource, raw_index);
|
||||
values[component] = EmitValueOrZeroIfCondition(
|
||||
state, EmitMemoryElementInBounds(state, resource, index),
|
||||
[&]() { return LoadWordInBounds(ctx, resource, index); });
|
||||
@@ -1052,9 +1054,8 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
|
||||
const auto address = component == 0u ? base
|
||||
: Binary(state, spv::OpIAdd, TypeU32(state), base,
|
||||
ConstantU32(state, component * 4u));
|
||||
const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
|
||||
ConstantU32(state, 2));
|
||||
const auto index = EmitMemoryElementIndex(state, resource, raw_index);
|
||||
EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index),
|
||||
[&]() {
|
||||
StoreWordInBounds(ctx, resource, index, ctx.Arg(inst, component + 1u));
|
||||
@@ -1112,7 +1113,7 @@ void DefineGetBdaPointer(EmitterState& state) {
|
||||
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
|
||||
const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64);
|
||||
const auto entry_pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferU64ElementPointer(state),
|
||||
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state, 64),
|
||||
entry_pointer, state.bda_pagetable_variable, ConstantU32(state, 0),
|
||||
page);
|
||||
const auto base = state.builder.AllocateId();
|
||||
@@ -1160,16 +1161,15 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
return EmitValueOrDefaultIfCondition(
|
||||
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
|
||||
const auto resource = PrepareStorageBufferResourceAccess(
|
||||
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
|
||||
const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
|
||||
ByteAddress(ctx, inst, mem), resource.byte_offset);
|
||||
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferPointer(state, 64));
|
||||
const auto byte_address = ByteAddress(ctx, inst, mem);
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
|
||||
ConstantU32(state, 3u));
|
||||
return EmitValueOrDefaultIfCondition(
|
||||
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
|
||||
ConstantU64(state, 0), [&]() {
|
||||
const auto pointer = EmitStorageBufferElementPointer(
|
||||
state, resource, index, TypeStorageBufferU64ElementPointer(state));
|
||||
state, resource, index, TypeStorageBufferElementPointer(state, 64));
|
||||
const auto old = EmitAtomicOperation(ctx, inst, pointer, spv::ScopeDevice);
|
||||
EmitAtomicMemoryBarrier(state, mem.kind);
|
||||
return old;
|
||||
@@ -1216,16 +1216,17 @@ void EmitSharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
const auto& mem = ctx.Memory(inst);
|
||||
const bool max_value = inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem));
|
||||
const auto index = DwordIndex(ctx, inst, mem);
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
EmitIfCondition(
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
|
||||
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
ctx.state.builder.AddFunction(spv::OpStore, ctx.scratch_u32_variable,
|
||||
ctx.Arg(inst, 1));
|
||||
const auto data = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), data,
|
||||
ctx.scratch_u32_variable);
|
||||
AtomicUpdate(
|
||||
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
|
||||
ctx.state, EmitMemoryElementPointer(ctx.state, resource, index),
|
||||
mem.kind, [&](uint32_t old) {
|
||||
const auto old_f =
|
||||
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
|
||||
@@ -1269,10 +1270,9 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
Binary(state, spv::OpIAdd, TypeU32(state), base, size),
|
||||
ConstantU32(state, 0xc000u)));
|
||||
}
|
||||
const auto raw_index =
|
||||
const auto index =
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto index = EmitMemoryElementIndex(state, access, raw_index);
|
||||
const auto exec = ctx.Arg(inst, 1);
|
||||
const auto ballot = ctx.Ballot(inst.Arg(1));
|
||||
const auto low = state.builder.AllocateId();
|
||||
@@ -1343,7 +1343,10 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
|
||||
}
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto element = EmitMemoryElementIndex(state, access, index);
|
||||
// Scalar buffer addressing aligns the base and each offset independently.
|
||||
const auto element = Binary(
|
||||
state, spv::OpIAdd, TypeU32(state), index,
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byte_offset, ConstantU32(state, 2)));
|
||||
const auto condition = EmitMemoryElementInBounds(state, access, element);
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
|
||||
return EmitNative<spv::OpLoad, IR::Type::U32>(
|
||||
|
||||
@@ -92,6 +92,15 @@ Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::
|
||||
return state.program.info.buffers[mem.resource].descriptor_format;
|
||||
}
|
||||
|
||||
uint32_t StorageBufferElementBits(const IR::Program& program, const IR::MemoryInfo& mem) {
|
||||
if (!mem.formatted) return mem.data_bits;
|
||||
const auto format = mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
|
||||
: program.info.buffers[mem.resource].descriptor_format;
|
||||
const auto info = Format::GetFormatInfo(format);
|
||||
return info.type == Format::ComponentType::Unknown || info.packed_bitfield
|
||||
? 32u : info.component_bits[0];
|
||||
}
|
||||
|
||||
void EmitMemoryOffsets(EmitterState& state) {
|
||||
for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
|
||||
const auto word =
|
||||
@@ -194,24 +203,18 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
|
||||
EXIT("physical address memory must use the BDA emitter\n");
|
||||
case IR::ResourceKind::ScalarBuffer:
|
||||
case IR::ResourceKind::Buffer: {
|
||||
const auto bits = mem.kind == IR::ResourceKind::Buffer
|
||||
? StorageBufferElementBits(state.program, mem) : 32u;
|
||||
const auto variable = bits == 8u ? state.storage_buffer_u8_variable
|
||||
: bits == 16u ? state.storage_buffer_u16_variable
|
||||
: state.storage_buffer_variable;
|
||||
access = PrepareStorageBufferResourceAccess(
|
||||
state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
|
||||
access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
|
||||
ConstantU32(state, 2u));
|
||||
access.add_index_offset = true;
|
||||
state, mem, variable, TypeStorageBufferPointer(state, bits));
|
||||
access.element_bits = bits;
|
||||
return access;
|
||||
}
|
||||
default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
|
||||
}
|
||||
access.length = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpArrayLength, TypeU32(state), access.length,
|
||||
access.object_pointer, 0);
|
||||
return access;
|
||||
}
|
||||
|
||||
uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
|
||||
uint32_t raw_index) {
|
||||
return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
|
||||
}
|
||||
|
||||
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
|
||||
@@ -240,7 +243,7 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
|
||||
return pointer;
|
||||
}
|
||||
return EmitStorageBufferElementPointer(state, access, index,
|
||||
TypeStorageBufferElementPointer(state));
|
||||
TypeStorageBufferElementPointer(state, access.element_bits));
|
||||
}
|
||||
|
||||
uint32_t EmitStorageBufferElementPointer(EmitterState& state,
|
||||
|
||||
@@ -123,46 +123,31 @@ uint32_t TypeFunction(EmitterState& state) {
|
||||
return state.function_type;
|
||||
}
|
||||
|
||||
uint32_t StorageRuntimeArrayType(EmitterState& state) {
|
||||
return state.builder.DecoratedType(
|
||||
spv::OpTypeRuntimeArray,
|
||||
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}}, TypeU32(state));
|
||||
uint32_t TypeStorageBufferElement(EmitterState& state, uint32_t bits) {
|
||||
return bits == 32u ? TypeU32(state) : bits == 64u ? TypeScalarU64(state)
|
||||
: state.builder.Type(spv::OpTypeInt, bits, 0);
|
||||
}
|
||||
|
||||
uint32_t StorageBufferType(EmitterState& state) {
|
||||
uint32_t StorageRuntimeArrayType(EmitterState& state, uint32_t bits) {
|
||||
return state.builder.DecoratedType(
|
||||
spv::OpTypeRuntimeArray,
|
||||
{{spv::OpDecorate, {spv::DecorationArrayStride, bits / 8u}}},
|
||||
TypeStorageBufferElement(state, bits));
|
||||
}
|
||||
|
||||
uint32_t StorageBufferType(EmitterState& state, uint32_t bits = 32) {
|
||||
return state.builder.DecoratedType(spv::OpTypeStruct,
|
||||
{{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
|
||||
{spv::OpDecorate, {spv::DecorationBlock}}},
|
||||
StorageRuntimeArrayType(state));
|
||||
StorageRuntimeArrayType(state, bits));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state));
|
||||
uint32_t TypeStorageBufferPointer(EmitterState& state, uint32_t bits) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state, bits));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state));
|
||||
}
|
||||
|
||||
uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
|
||||
return state.builder.DecoratedType(
|
||||
spv::OpTypeRuntimeArray,
|
||||
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}}, TypeScalarU64(state));
|
||||
}
|
||||
|
||||
uint32_t StorageBufferU64Type(EmitterState& state) {
|
||||
return state.builder.DecoratedType(spv::OpTypeStruct,
|
||||
{{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
|
||||
{spv::OpDecorate, {spv::DecorationBlock}}},
|
||||
StorageU64RuntimeArrayType(state));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state));
|
||||
}
|
||||
|
||||
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
|
||||
uint32_t TypeStorageBufferElementPointer(EmitterState& state, uint32_t bits) {
|
||||
return TypePointer(state, spv::StorageClassStorageBuffer, TypeStorageBufferElement(state, bits));
|
||||
}
|
||||
|
||||
uint32_t TypePhysicalU32Pointer(EmitterState& state) {
|
||||
@@ -244,27 +229,34 @@ void DefineDescriptors(EmitterState& state) {
|
||||
case IR::DescriptorBindingKind::Buffers:
|
||||
state.storage_buffer_variable =
|
||||
Define(ArrayType(StorageBufferType(state)), "buffers");
|
||||
if (state.requirements.buffer_u8) {
|
||||
state.storage_buffer_u8_variable =
|
||||
Define(ArrayType(StorageBufferType(state, 8)), "buffers_u8");
|
||||
}
|
||||
if (state.requirements.buffer_u16) {
|
||||
state.storage_buffer_u16_variable =
|
||||
Define(ArrayType(StorageBufferType(state, 16)), "buffers_u16");
|
||||
}
|
||||
if (state.requirements.buffer_int64_atomics) {
|
||||
state.storage_buffer_u64_variable =
|
||||
Define(ArrayType(StorageBufferU64Type(state)), "buffers_u64");
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
|
||||
spv::DecorationAliased);
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
|
||||
spv::DecorationAliased);
|
||||
Define(ArrayType(StorageBufferType(state, 64)), "buffers_u64");
|
||||
}
|
||||
if (state.requirements.coherent_buffers) {
|
||||
for (const auto variable: {state.storage_buffer_variable, state.storage_buffer_u8_variable,
|
||||
state.storage_buffer_u16_variable, state.storage_buffer_u64_variable}) {
|
||||
if (variable == 0) continue;
|
||||
if (state.storage_buffer_u8_variable != 0 || state.storage_buffer_u16_variable != 0 ||
|
||||
state.storage_buffer_u64_variable != 0) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased);
|
||||
}
|
||||
// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
|
||||
// must participate in visibility for cache-bypassing polling loads.
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
|
||||
spv::DecorationCoherent);
|
||||
if (state.storage_buffer_u64_variable != 0) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
|
||||
spv::DecorationCoherent);
|
||||
if (state.requirements.coherent_buffers) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, variable, spv::DecorationCoherent);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case IR::DescriptorBindingKind::BdaPagetable:
|
||||
state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");
|
||||
state.bda_pagetable_variable = Define(StorageBufferType(state, 64), "bda_pagetable");
|
||||
break;
|
||||
case IR::DescriptorBindingKind::FaultBuffer:
|
||||
state.fault_buffer_variable = Define(StorageBufferType(state), "fault_buffer");
|
||||
@@ -669,6 +661,13 @@ void DefineModule(EmitterState& state) {
|
||||
|
||||
state.builder.RequireCapability(spv::CapabilityShader);
|
||||
state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
|
||||
if (state.requirements.buffer_u8) {
|
||||
state.builder.RequireExtension("SPV_KHR_8bit_storage");
|
||||
state.builder.RequireCapability(spv::CapabilityStorageBuffer8BitAccess);
|
||||
}
|
||||
if (state.requirements.buffer_u16) {
|
||||
state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
|
||||
}
|
||||
if (state.program.info.uses_dma) {
|
||||
state.builder.RequireCapability(spv::CapabilityInt64);
|
||||
state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
|
||||
|
||||
Reference in New Issue
Block a user